Data Pre-processing

Load needed libraries

library(readr)
library(ggplot2)
library(dplyr)
library(caret)
library(glmnet)
library(boot)
library(tree)
library(ranger)
library(xgboost)
library(gbm)
library(vip)
library(ISLR)
library(tidyr)
library(gridExtra)
library(reshape2)

Set the seed for reproducibility

set.seed(1)

Load the dataset

original_lc_data <- read.csv("LCdata.csv",sep = ";")
lc_data <- original_lc_data

Remove attributes not available for prediction

lc_data <- subset(lc_data, select = -c(collection_recovery_fee, installment, issue_d,
                                       last_pymnt_amnt, last_pymnt_d, loan_status,
                                       next_pymnt_d, out_prncp, out_prncp_inv,
                                       pymnt_plan, recoveries, total_pymnt,
                                       total_pymnt_inv,total_rec_int, total_rec_late_fee, 
                                       total_rec_prncp))
summary(lc_data)
       id             member_id          loan_amnt      funded_amnt    funded_amnt_inv     term          
 Min.   :   54734   Min.   :   70473   Min.   :  500   Min.   :  500   Min.   :    0   Length:798641     
 1st Qu.: 9207230   1st Qu.:10877939   1st Qu.: 8000   1st Qu.: 8000   1st Qu.: 8000   Class :character  
 Median :34433372   Median :37095300   Median :13000   Median :13000   Median :13000   Mode  :character  
 Mean   :32463636   Mean   :35000265   Mean   :14754   Mean   :14741   Mean   :14702                     
 3rd Qu.:54900100   3rd Qu.:58470266   3rd Qu.:20000   3rd Qu.:20000   3rd Qu.:20000                     
 Max.   :68617057   Max.   :73544841   Max.   :35000   Max.   :35000   Max.   :35000                     
                                                                                                         
    int_rate      emp_title          emp_length        home_ownership       annual_inc      verification_status
 Min.   : 5.32   Length:798641      Length:798641      Length:798641      Min.   :      0   Length:798641      
 1st Qu.: 9.99   Class :character   Class :character   Class :character   1st Qu.:  45000   Class :character   
 Median :12.99   Mode  :character   Mode  :character   Mode  :character   Median :  65000   Mode  :character   
 Mean   :13.24                                                            Mean   :  75014                      
 3rd Qu.:16.20                                                            3rd Qu.:  90000                      
 Max.   :28.99                                                            Max.   :9500000                      
                                                                          NA's   :4                            
     url                desc             purpose             title             zip_code          addr_state       
 Length:798641      Length:798641      Length:798641      Length:798641      Length:798641      Length:798641     
 Class :character   Class :character   Class :character   Class :character   Class :character   Class :character  
 Mode  :character   Mode  :character   Mode  :character   Mode  :character   Mode  :character   Mode  :character  
                                                                                                                  
                                                                                                                  
                                                                                                                  
                                                                                                                  
      dti           delinq_2yrs      earliest_cr_line   inq_last_6mths    mths_since_last_delinq mths_since_last_record
 Min.   :   0.00   Min.   : 0.0000   Length:798641      Min.   : 0.0000   Min.   :  0.0          Min.   :  0.0         
 1st Qu.:  11.91   1st Qu.: 0.0000   Class :character   1st Qu.: 0.0000   1st Qu.: 15.0          1st Qu.: 51.0         
 Median :  17.66   Median : 0.0000   Mode  :character   Median : 0.0000   Median : 31.0          Median : 70.0         
 Mean   :  18.16   Mean   : 0.3145                      Mean   : 0.6947   Mean   : 34.1          Mean   : 70.1         
 3rd Qu.:  23.95   3rd Qu.: 0.0000                      3rd Qu.: 1.0000   3rd Qu.: 50.0          3rd Qu.: 92.0         
 Max.   :9999.00   Max.   :39.0000                      Max.   :33.0000   Max.   :188.0          Max.   :129.0         
                   NA's   :25                           NA's   :25        NA's   :408818         NA's   :675190        
    open_acc        pub_rec          revol_bal         revol_util       total_acc      initial_list_status
 Min.   : 0.00   Min.   : 0.0000   Min.   :      0   Min.   :  0.00   Min.   :  1.00   Length:798641      
 1st Qu.: 8.00   1st Qu.: 0.0000   1st Qu.:   6443   1st Qu.: 37.70   1st Qu.: 17.00   Class :character   
 Median :11.00   Median : 0.0000   Median :  11876   Median : 56.00   Median : 24.00   Mode  :character   
 Mean   :11.55   Mean   : 0.1953   Mean   :  16930   Mean   : 55.05   Mean   : 25.27                      
 3rd Qu.:14.00   3rd Qu.: 0.0000   3rd Qu.:  20839   3rd Qu.: 73.50   3rd Qu.: 32.00                      
 Max.   :90.00   Max.   :63.0000   Max.   :2904836   Max.   :892.30   Max.   :169.00                      
 NA's   :25      NA's   :25        NA's   :2         NA's   :454      NA's   :25                          
 last_credit_pull_d collections_12_mths_ex_med mths_since_last_major_derog  policy_code application_type  
 Length:798641      Min.   : 0.00000           Min.   :  0.0               Min.   :1    Length:798641     
 Class :character   1st Qu.: 0.00000           1st Qu.: 27.0               1st Qu.:1    Class :character  
 Mode  :character   Median : 0.00000           Median : 44.0               Median :1    Mode  :character  
                    Mean   : 0.01447           Mean   : 44.1               Mean   :1                      
                    3rd Qu.: 0.00000           3rd Qu.: 61.0               3rd Qu.:1                      
                    Max.   :20.00000           Max.   :188.0               Max.   :1                      
                    NA's   :126                NA's   :599107                                             
 annual_inc_joint   dti_joint      verification_status_joint acc_now_delinq       tot_coll_amt      tot_cur_bal     
 Min.   : 17950   Min.   : 3.0     Length:798641             Min.   : 0.000000   Min.   :      0   Min.   :      0  
 1st Qu.: 76167   1st Qu.:13.3     Class :character          1st Qu.: 0.000000   1st Qu.:      0   1st Qu.:  29861  
 Median :101886   Median :17.7     Mode  :character          Median : 0.000000   Median :      0   Median :  80647  
 Mean   :110745   Mean   :18.4                               Mean   : 0.005026   Mean   :    228   Mean   : 139508  
 3rd Qu.:133000   3rd Qu.:22.6                               3rd Qu.: 0.000000   3rd Qu.:      0   3rd Qu.: 208229  
 Max.   :500000   Max.   :43.9                               Max.   :14.000000   Max.   :9152545   Max.   :8000078  
 NA's   :798181   NA's   :798183                             NA's   :25          NA's   :63276     NA's   :63276    
  open_acc_6m       open_il_6m      open_il_12m      open_il_24m     mths_since_rcnt_il  total_bal_il   
 Min.   : 0.0     Min.   : 0.0     Min.   : 0.0     Min.   : 0.0     Min.   :  0.0      Min.   :     0  
 1st Qu.: 0.0     1st Qu.: 1.0     1st Qu.: 0.0     1st Qu.: 0.0     1st Qu.:  6.0      1st Qu.: 10164  
 Median : 1.0     Median : 2.0     Median : 0.0     Median : 1.0     Median : 12.0      Median : 24544  
 Mean   : 1.1     Mean   : 2.9     Mean   : 0.8     Mean   : 1.7     Mean   : 21.1      Mean   : 36428  
 3rd Qu.: 2.0     3rd Qu.: 4.0     3rd Qu.: 1.0     3rd Qu.: 2.0     3rd Qu.: 23.0      3rd Qu.: 47640  
 Max.   :14.0     Max.   :33.0     Max.   :12.0     Max.   :19.0     Max.   :363.0      Max.   :878459  
 NA's   :779525   NA's   :779525   NA's   :779525   NA's   :779525   NA's   :780030     NA's   :779525  
    il_util        open_rv_12m      open_rv_24m       max_bal_bc        all_util      total_rev_hi_lim 
 Min.   :  0.0    Min.   : 0.0     Min.   : 0       Min.   :    0    Min.   :  0.0    Min.   :      0  
 1st Qu.: 58.4    1st Qu.: 0.0     1st Qu.: 1       1st Qu.: 2406    1st Qu.: 47.6    1st Qu.:  13900  
 Median : 74.8    Median : 1.0     Median : 2       Median : 4502    Median : 61.9    Median :  23700  
 Mean   : 71.5    Mean   : 1.4     Mean   : 3       Mean   : 5878    Mean   : 60.8    Mean   :  32093  
 3rd Qu.: 87.7    3rd Qu.: 2.0     3rd Qu.: 4       3rd Qu.: 7774    3rd Qu.: 75.2    3rd Qu.:  39800  
 Max.   :223.3    Max.   :22.0     Max.   :43       Max.   :83047    Max.   :151.4    Max.   :9999999  
 NA's   :782007   NA's   :779525   NA's   :779525   NA's   :779525   NA's   :779525   NA's   :63276    
     inq_fi        total_cu_tl      inq_last_12m   
 Min.   : 0.0     Min.   : 0.0     Min.   :-4      
 1st Qu.: 0.0     1st Qu.: 0.0     1st Qu.: 0      
 Median : 0.0     Median : 0.0     Median : 2      
 Mean   : 0.9     Mean   : 1.5     Mean   : 2      
 3rd Qu.: 1.0     3rd Qu.: 2.0     3rd Qu.: 3      
 Max.   :16.0     Max.   :35.0     Max.   :32      
 NA's   :779525   NA's   :779525   NA's   :779525  

First we delete the columns which aren’t useful for our prediction

lc_data$id <- NULL
lc_data$member_id <- NULL
lc_data$zip_code <- NULL
lc_data$url <- NULL

Looks like policy_code contains just value equal to 1, it can be removed

lc_data$policy_code <- NULL

Remove additional columns which are related to the historical data

lc_data$last_credit_pull_d <- NULL

Then we delete the columns which can’t be converted to categorical and require NLP

lc_data$title <- NULL
lc_data$desc <- NULL
lc_data$emp_title <- NULL

Let’s examine the loan_amnt column

sum(is.na(lc_data$loan_amnt))
[1] 0
cor(lc_data$loan_amnt, lc_data$int_rate)
[1] 0.1447189
hist(lc_data$loan_amnt, breaks = 20, main = "loan_amnt distribution", xlab = "loan_amnt", col = "lightblue", border = "black")

ggplot(data = lc_data, mapping = aes(x=int_rate,y=loan_amnt)) + geom_boxplot()

Standardize loan_amnt

#lc_data$loan_amnt <- scale(lc_data$loan_amnt)

Let’s examine the funded_amnt column

sum(is.na(lc_data$funded_amnt))
[1] 0
cor(lc_data$funded_amnt, lc_data$int_rate)
[1] 0.1448634
hist(lc_data$funded_amnt, breaks = 20, main = "funded_amnt distribution", xlab = "funded_amnt", col = "lightblue", border = "black")

As we can see, funded_amnt is almost the same as the loan_amnt column, consequently, we remove it.

lc_data$funded_amnt <- NULL 

Let’s examine the funded_amnt_inv column

sum(is.na(lc_data$funded_amnt_inv))
[1] 0
cor(lc_data$funded_amnt_inv, lc_data$int_rate)
[1] 0.1449083
hist(lc_data$funded_amnt_inv, breaks = 20, main = "funded_amnt_inv distribution", xlab = "funded_amnt_inv", col = "lightblue", border = "black")

Remove funded_amnt_inv for the same reason as above

lc_data$funded_amnt_inv <- NULL

Let’s see the int_rate distribution.

hist(lc_data$int_rate, breaks = 20, main = "int_rate distribution", xlab = "int_rate", col = "lightblue", border = "black")

Standardize int rate:

#lc_data$int_rate <- scale(lc_data$int_rate)

As we can observe, there are 40363 NAs. We can assume 40363 do not work.

barplot(table(lc_data$emp_length),
        xlab = "emp_length years", 
        ylab = "Frequency", 
        col = "skyblue", 
        border = "black",
        cex.names = 0.6)  # The size of the main title

Since emp_length seems to be categorical, we transform it to as a factor and then as numeric. The conversion to numeric is needed for supporting the Decision Trees and XGBoost

lc_data$emp_length <- as.factor(lc_data$emp_length)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=emp_length)) + geom_boxplot()

lc_data$emp_length <- as.numeric(lc_data$emp_length)

As we can see, term plays a crucial role in predicting the interest rate.

lc_data$term <- as.factor(lc_data$term)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=term)) + geom_boxplot()

lc_data$term <- as.numeric(lc_data$term)

Cleaning of home_ownership:

During the data cleaning phase, our analysis revealed that the variable “home_ownership” does not show a distinct correlation with interest rates. Specifically, among the categories, “ANY” and “OTHER” contain 2 and 154 cases, respectively, while the “NONE” category comprises 39 cases. Although the “NONE” category appears to demonstrate a higher interest rate compared to others, the limited sample size of 39 cases raises doubts about the reliability of this observation. Notably, the “NONE” category might pertain to individuals experiencing homelessness, prompting ethical concerns about loan provision to this demographic.

table(lc_data$home_ownership)

     ANY MORTGAGE     NONE    OTHER      OWN     RENT 
       2   399151       45      155    78789   320499 
ggplot(data = lc_data, mapping = aes(x=int_rate,y=home_ownership)) + geom_boxplot()

Then, we retain mortgage, own and rent:

lc_data <- lc_data %>% filter(home_ownership %in% c("MORTGAGE","OWN","RENT"))
lc_data$home_ownership <- as.numeric(as.factor(lc_data$home_ownership))

Most of the loan applications are Individual, this means that most of the values of the columns dti_joint,annual_inc_joint and verification_status_joint are Null. We would like to keep the information about Joint loans, this means that we can replace the Null values with 0.

nav <- c('', ' ')
lc_data <- transform(lc_data, verification_status_joint=replace(verification_status_joint, verification_status_joint %in% nav, NA))
lc_data <-
  lc_data %>%
  mutate(dti_joint = ifelse(is.na(dti_joint) == TRUE, 0, dti_joint)) %>%
  mutate(annual_inc_joint = ifelse(is.na(annual_inc_joint) == TRUE, 0, annual_inc_joint)) %>%
  mutate(verification_status_joint = ifelse(is.na(verification_status_joint) == TRUE, 'NA', verification_status_joint))

The empty string or null value in verification_status_joint is replaced successfully.

table(lc_data$verification_status)

   Not Verified Source Verified        Verified 
         240255          296631          261553 
table(lc_data$verification_status_joint)

             NA    Not Verified Source Verified        Verified 
         797979             253              53             154 

Then verification_status_joint and verification_status columns are converted in categorical and then numerical value. The column application_type is obsolete, since the information about whether the loan is individual or joint is already contained in the previous variables.

lc_data$verification_status <- as.numeric(as.factor(lc_data$verification_status))
lc_data$verification_status_joint <- as.numeric(as.factor(lc_data$verification_status_joint))
lc_data <- lc_data %>% select(-application_type)

Let’s check if other is NA or a real value for purpose. It’s a real one, so we don’t have to handle it.

lc_data$purpose <- as.factor(lc_data$purpose)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=purpose)) + geom_boxplot()

lc_data$purpose <- as.numeric(lc_data$purpose)

Let’s have a glance to the state address:

table(lc_data$addr_state)

    AK     AL     AR     AZ     CA     CO     CT     DC     DE     FL     GA     HI     IA     ID     IL     IN     KS 
  1992  10101   5953  18359 116578  16934  12154   2188   2268  54819  26146   4112     13     11  31880  12393   7105 
    KY     LA     MA     MD     ME     MI     MN     MO     MS     MT     NC     ND     NE     NH     NJ     NM     NV 
  7726   9498  18546  18906    469  20678  14306  12821   3455   2286  22135    431   1064   3865  29991   4428  11155 
    NY     OH     OK     OR     PA     RI     SC     SD     TN     TX     UT     VA     VT     WA     WI     WV     WY 
 66790  26682   7266   9806  28221   3499   9609   1615  11618  63982   5629  23616   1606  17470  10446   3977   1841 
lc_data$addr_state <- as.factor(lc_data$addr_state)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=addr_state)) + geom_boxplot()

lc_data$addr_state <- as.numeric(lc_data$addr_state)

Regarding delinquency in the last 2 years, there are few NAs then remove them:

lc_data <- lc_data %>% 
    filter(!(is.na(delinq_2yrs)))

The columns mths_since_delinq_cat, mths_since_last_record, mths_since_rcnt_il and mths_since_last_major_derog contain numerical values which refer to the number of the months. Since this columns contain a lot of null values which can’t be replaced with 0’s, one of the most appropriate operations that can be made is applying discretization. We do this by creating a set of contiguous bins based on years, while for the null values we create a separate bin.

lc_data <- lc_data %>%
  mutate(mths_since_delinq_cat = ifelse(
    is.na(mths_since_last_delinq) == TRUE,
    "NONE",
    ifelse(
      mths_since_last_delinq <= 12,
      "Less_1_Y",
      ifelse(
        mths_since_last_delinq <= 24,
        "Less_2_Y",
        ifelse(
          mths_since_last_delinq <= 36,
          "Less_3_Y",
          ifelse(mths_since_last_delinq <= 48, "Less_4_Y", "More_4_Y")
        )
      )
    )
  )) %>% select(-mths_since_last_delinq)
          
lc_data$mths_since_delinq_cat <- as.factor(lc_data$mths_since_delinq_cat)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=mths_since_delinq_cat))+geom_boxplot()

lc_data$mths_since_delinq_cat <- as.numeric(lc_data$mths_since_delinq_cat)
lc_data <- lc_data %>%
  mutate(mths_since_last_record_cat = ifelse(
    is.na(mths_since_last_record) == TRUE,
    "NONE",
    ifelse(
      mths_since_last_record <= 12,
      "Less_1_Y",
      ifelse(
        mths_since_last_record <= 24,
        "Less_2_Y",
        ifelse(
          mths_since_last_record <= 36,
          "Less_3_Y",
          ifelse(mths_since_last_record <= 48, "Less_4_Y", "More_4_Y")
        )
      )
    )
  )) %>% select(-mths_since_last_record)

lc_data$mths_since_last_record_cat <- as.factor(lc_data$mths_since_last_record_cat)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=mths_since_last_record_cat))+geom_boxplot()

lc_data$mths_since_last_record_cat <- as.numeric(lc_data$mths_since_last_record_cat)
lc_data <-lc_data %>% 
  mutate(mths_since_rcnt_il_cat =  ifelse(
    is.na(mths_since_rcnt_il) == TRUE,
    "NONE",
    ifelse(
      mths_since_rcnt_il <= 12,
      "Less_1_Y",
      ifelse(
        mths_since_rcnt_il <= 24,
        "Less_2_Y",
        ifelse(
          mths_since_rcnt_il <= 36,
          "Less_3_Y",
          ifelse(mths_since_rcnt_il <= 48, "Less_4_Y", "More_4_Y")
        )
      )
    )
  )) %>% select(-mths_since_rcnt_il)

lc_data$mths_since_rcnt_il_cat <- as.factor(lc_data$mths_since_rcnt_il_cat)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=mths_since_rcnt_il_cat))+geom_boxplot()

lc_data$mths_since_rcnt_il_cat <- as.numeric(lc_data$mths_since_rcnt_il_cat)
lc_data <-lc_data %>% 
  mutate(mths_since_last_major_derog_cat =  ifelse(
    is.na(mths_since_last_major_derog) == TRUE,
    "NONE",
    ifelse(
      mths_since_last_major_derog <= 12,
      "Less_1_Y",
      ifelse(
        mths_since_last_major_derog <= 24,
        "Less_2_Y",
        ifelse(
          mths_since_last_major_derog <= 36,
          "Less_3_Y",
          ifelse(mths_since_last_major_derog <= 48, "Less_4_Y", "More_4_Y")
        )
      )
    )
  )) %>% select(-mths_since_last_major_derog)

lc_data$mths_since_last_major_derog_cat <- as.factor(lc_data$mths_since_last_major_derog_cat)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=mths_since_last_major_derog_cat))+geom_boxplot()

lc_data$mths_since_last_major_derog_cat <- as.numeric(lc_data$mths_since_last_major_derog_cat)

The variable initial_list_status identifies whether a loan was initially listed in the whole (W) or fractional (F) market. This variable could be useful so we can keep it and transform it to a factor and then to a numeric value, for the same purpose of compatibility with the XGBoost function.

lc_data$initial_list_status <- as.factor(lc_data$initial_list_status)
ggplot(data = lc_data, mapping = aes(x=int_rate,y=initial_list_status))+geom_boxplot()

lc_data$initial_list_status <- as.numeric(lc_data$initial_list_status)

Let’s check which columns still have null values

colSums(is.na(lc_data))
                      loan_amnt                            term                        int_rate 
                              0                               0                               0 
                     emp_length                  home_ownership                      annual_inc 
                              0                               0                               0 
            verification_status                         purpose                      addr_state 
                              0                               0                               0 
                            dti                     delinq_2yrs                earliest_cr_line 
                              0                               0                               0 
                 inq_last_6mths                        open_acc                         pub_rec 
                              0                               0                               0 
                      revol_bal                      revol_util                       total_acc 
                              2                             428                               0 
            initial_list_status      collections_12_mths_ex_med                annual_inc_joint 
                              0                              99                               0 
                      dti_joint       verification_status_joint                  acc_now_delinq 
                              0                               0                               0 
                   tot_coll_amt                     tot_cur_bal                     open_acc_6m 
                          63132                           63132                          779302 
                     open_il_6m                     open_il_12m                     open_il_24m 
                         779302                          779302                          779302 
                   total_bal_il                         il_util                     open_rv_12m 
                         779302                          781784                          779302 
                    open_rv_24m                      max_bal_bc                        all_util 
                         779302                          779302                          779302 
               total_rev_hi_lim                          inq_fi                     total_cu_tl 
                          63132                          779302                          779302 
                   inq_last_12m           mths_since_delinq_cat      mths_since_last_record_cat 
                         779302                               0                               0 
         mths_since_rcnt_il_cat mths_since_last_major_derog_cat 
                              0                               0 

The columns revol_bal and revol_util contain only few NA values, those values can’t be replaced with 0, then we filter the values which are not NAs.

lc_data <- lc_data %>% 
    filter(!(is.na(revol_bal))) %>% 
        filter(!(is.na(revol_util)))

Let’s check which columns still have null values:

names(which(colSums(is.na(lc_data)) > 0))
 [1] "collections_12_mths_ex_med" "tot_coll_amt"               "tot_cur_bal"                "open_acc_6m"               
 [5] "open_il_6m"                 "open_il_12m"                "open_il_24m"                "total_bal_il"              
 [9] "il_util"                    "open_rv_12m"                "open_rv_24m"                "max_bal_bc"                
[13] "all_util"                   "total_rev_hi_lim"           "inq_fi"                     "total_cu_tl"               
[17] "inq_last_12m"              

Replace null values with 0 where is possible

lc_data <-
  lc_data %>%
  mutate(open_acc_6m = ifelse(is.na(open_acc_6m) == TRUE, 0, open_acc_6m)) %>%
  mutate(tot_cur_bal = ifelse(is.na(tot_cur_bal) == TRUE, 0, tot_cur_bal)) %>%
  mutate(open_il_6m = ifelse(is.na(open_il_6m) == TRUE, 0, open_il_6m)) %>%
  mutate(open_il_12m = ifelse(is.na(open_il_12m) == TRUE, 0, open_il_12m)) %>%
  mutate(open_il_24m = ifelse(is.na(open_il_24m) == TRUE, 0, open_il_24m)) %>%
  mutate(total_bal_il = ifelse(is.na(total_bal_il) == TRUE, 0, total_bal_il)) %>%
  mutate(il_util = ifelse(is.na(il_util) == TRUE, 0, il_util)) %>%
  mutate(open_rv_12m = ifelse(is.na(open_rv_12m) == TRUE, 0, open_rv_12m)) %>%
  mutate(total_rev_hi_lim = ifelse(is.na(total_rev_hi_lim) == TRUE, 0, total_rev_hi_lim)) %>%
  mutate(max_bal_bc = ifelse(is.na(max_bal_bc) == TRUE, 0, max_bal_bc)) %>%
  mutate(all_util = ifelse(is.na(all_util) == TRUE, 0, all_util)) %>%
  mutate(inq_fi = ifelse(is.na(inq_fi) == TRUE, 0, inq_fi)) %>%
  mutate(total_cu_tl = ifelse(is.na(total_cu_tl) == TRUE, 0, total_cu_tl)) %>%
  mutate(inq_last_12m = ifelse(is.na(inq_last_12m) == TRUE, 0, inq_last_12m)) %>%
  mutate(open_rv_24m = ifelse(is.na(open_rv_24m) == TRUE, 0, open_rv_24m)) %>%
  mutate(tot_coll_amt = ifelse(is.na(tot_coll_amt)== TRUE,0, tot_coll_amt)) %>%
  mutate(collections_12_mths_ex_med = ifelse(is.na(collections_12_mths_ex_med)== TRUE,0, collections_12_mths_ex_med))

earliest_cr_line contains the month the borrower’s earliest reported credit line was opened. Even if this date consists only on month and year, still there are too many unique values. We could transform the dates in to a numerical value, by converting them from date into Unix Time. This unit measures time by the number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970. Since this column doesn’t contain the day number, we take as a reference the first day of the month.

lc_data <- lc_data %>% 
    filter(!(is.na(earliest_cr_line)))

# function to replace dates with unix time
to_unix_time <- function(date) {
  tmp <- paste("01", date, sep="-")
  return (as.numeric(as.POSIXct(tmp, format="%d-%b-%Y", tz="UTC")))
}

# map dates to unix time
lc_data$earliest_cr_line <- apply(lc_data, 1, function(row) to_unix_time(row["earliest_cr_line"]))

# standardize them
#lc_data$earliest_cr_line <- scale(lc_data$earliest_cr_line)

Outliers Removal:

boxplot(lc_data$int_rate)

# Identify outliers using boxplot
outliers <- boxplot(lc_data$int_rate, plot = FALSE)$out
# Remove outliers from the dataset
lc_data <- lc_data[!lc_data$int_rate %in% outliers, ]
summary(lc_data)
   loan_amnt          term          int_rate       emp_length    home_ownership    annual_inc      verification_status
 Min.   :  500   Min.   :1.000   Min.   : 5.32   Min.   : 1.00   Min.   :1.000   Min.   :      0   Min.   :1.000      
 1st Qu.: 8000   1st Qu.:1.000   1st Qu.: 9.99   1st Qu.: 3.00   1st Qu.:1.000   1st Qu.:  45000   1st Qu.:1.000      
 Median :13000   Median :1.000   Median :12.99   Median : 4.00   Median :1.000   Median :  65000   Median :2.000      
 Mean   :14718   Mean   :1.296   Mean   :13.15   Mean   : 5.11   Mean   :1.901   Mean   :  75007   Mean   :2.023      
 3rd Qu.:20000   3rd Qu.:2.000   3rd Qu.:15.95   3rd Qu.: 7.00   3rd Qu.:3.000   3rd Qu.:  90000   3rd Qu.:3.000      
 Max.   :35000   Max.   :2.000   Max.   :25.28   Max.   :12.00   Max.   :3.000   Max.   :9500000   Max.   :3.000      
    purpose         addr_state         dti           delinq_2yrs      earliest_cr_line     inq_last_6mths   
 Min.   : 1.000   Min.   : 1.00   Min.   :   0.00   Min.   : 0.0000   Min.   :-820540800   Min.   : 0.0000  
 1st Qu.: 3.000   1st Qu.:10.00   1st Qu.:  11.90   1st Qu.: 0.0000   1st Qu.: 770428800   1st Qu.: 0.0000  
 Median : 3.000   Median :24.00   Median :  17.64   Median : 0.0000   Median : 933465600   Median : 0.0000  
 Mean   : 3.562   Mean   :24.14   Mean   :  18.15   Mean   : 0.3138   Mean   : 888844650   Mean   : 0.6899  
 3rd Qu.: 3.000   3rd Qu.:37.00   3rd Qu.:  23.93   3rd Qu.: 0.0000   3rd Qu.:1051747200   3rd Qu.: 1.0000  
 Max.   :14.000   Max.   :51.00   Max.   :9999.00   Max.   :39.0000   Max.   :1351728000   Max.   :33.0000  
    open_acc        pub_rec         revol_bal         revol_util       total_acc      initial_list_status
 Min.   : 1.00   Min.   : 0.000   Min.   :      0   Min.   :  0.00   Min.   :  1.00   Min.   :1.000      
 1st Qu.: 8.00   1st Qu.: 0.000   1st Qu.:   6456   1st Qu.: 37.60   1st Qu.: 17.00   1st Qu.:1.000      
 Median :11.00   Median : 0.000   Median :  11888   Median : 55.90   Median : 24.00   Median :1.000      
 Mean   :11.55   Mean   : 0.195   Mean   :  16943   Mean   : 55.02   Mean   : 25.27   Mean   :1.486      
 3rd Qu.:14.00   3rd Qu.: 0.000   3rd Qu.:  20849   3rd Qu.: 73.50   3rd Qu.: 32.00   3rd Qu.:2.000      
 Max.   :90.00   Max.   :63.000   Max.   :2904836   Max.   :892.30   Max.   :169.00   Max.   :2.000      
 collections_12_mths_ex_med annual_inc_joint     dti_joint        verification_status_joint acc_now_delinq    
 Min.   : 0.00000           Min.   :     0.0   Min.   : 0.00000   Min.   :1.000             Min.   : 0.00000  
 1st Qu.: 0.00000           1st Qu.:     0.0   1st Qu.: 0.00000   1st Qu.:1.000             1st Qu.: 0.00000  
 Median : 0.00000           Median :     0.0   Median : 0.00000   Median :1.000             Median : 0.00000  
 Mean   : 0.01444           Mean   :    63.2   Mean   : 0.01032   Mean   :1.001             Mean   : 0.00498  
 3rd Qu.: 0.00000           3rd Qu.:     0.0   3rd Qu.: 0.00000   3rd Qu.:1.000             3rd Qu.: 0.00000  
 Max.   :20.00000           Max.   :500000.0   Max.   :43.86000   Max.   :4.000             Max.   :14.00000  
  tot_coll_amt      tot_cur_bal       open_acc_6m        open_il_6m        open_il_12m        open_il_24m      
 Min.   :      0   Min.   :      0   Min.   : 0.0000   Min.   : 0.00000   Min.   : 0.00000   Min.   : 0.00000  
 1st Qu.:      0   1st Qu.:  23128   1st Qu.: 0.0000   1st Qu.: 0.00000   1st Qu.: 0.00000   1st Qu.: 0.00000  
 Median :      0   Median :  65355   Median : 0.0000   Median : 0.00000   Median : 0.00000   Median : 0.00000  
 Mean   :    210   Mean   : 128454   Mean   : 0.0263   Mean   : 0.06979   Mean   : 0.01803   Mean   : 0.03974  
 3rd Qu.:      0   3rd Qu.: 195918   3rd Qu.: 0.0000   3rd Qu.: 0.00000   3rd Qu.: 0.00000   3rd Qu.: 0.00000  
 Max.   :9152545   Max.   :8000078   Max.   :14.0000   Max.   :33.00000   Max.   :12.00000   Max.   :15.00000  
  total_bal_il         il_util         open_rv_12m        open_rv_24m         max_bal_bc         all_util      
 Min.   :     0.0   Min.   :  0.000   Min.   : 0.00000   Min.   : 0.00000   Min.   :    0.0   Min.   :  0.000  
 1st Qu.:     0.0   1st Qu.:  0.000   1st Qu.: 0.00000   1st Qu.: 0.00000   1st Qu.:    0.0   1st Qu.:  0.000  
 Median :     0.0   Median :  0.000   Median : 0.00000   Median : 0.00000   Median :    0.0   Median :  0.000  
 Mean   :   872.1   Mean   :  1.488   Mean   : 0.03307   Mean   : 0.07102   Mean   :  141.1   Mean   :  1.456  
 3rd Qu.:     0.0   3rd Qu.:  0.000   3rd Qu.: 0.00000   3rd Qu.: 0.00000   3rd Qu.:    0.0   3rd Qu.:  0.000  
 Max.   :878459.0   Max.   :200.000   Max.   :22.00000   Max.   :43.00000   Max.   :83047.0   Max.   :151.400  
 total_rev_hi_lim      inq_fi          total_cu_tl        inq_last_12m      mths_since_delinq_cat
 Min.   :      0   Min.   : 0.00000   Min.   : 0.00000   Min.   :-4.00000   Min.   :1.000        
 1st Qu.:  11700   1st Qu.: 0.00000   1st Qu.: 0.00000   1st Qu.: 0.00000   1st Qu.:3.000        
 Median :  21800   Median : 0.00000   Median : 0.00000   Median : 0.00000   Median :6.000        
 Mean   :  29595   Mean   : 0.02251   Mean   : 0.03675   Mean   : 0.04707   Mean   :4.577        
 3rd Qu.:  37900   3rd Qu.: 0.00000   3rd Qu.: 0.00000   3rd Qu.: 0.00000   3rd Qu.:6.000        
 Max.   :9999999   Max.   :16.00000   Max.   :35.00000   Max.   :32.00000   Max.   :6.000        
 mths_since_last_record_cat mths_since_rcnt_il_cat mths_since_last_major_derog_cat
 Min.   :1.000              Min.   :1.000          Min.   :1.000                  
 1st Qu.:6.000              1st Qu.:6.000          1st Qu.:6.000                  
 Median :6.000              Median :6.000          Median :6.000                  
 Mean   :5.779              Mean   :5.906          Mean   :5.436                  
 3rd Qu.:6.000              3rd Qu.:6.000          3rd Qu.:6.000                  
 Max.   :6.000              Max.   :6.000          Max.   :6.000                  

Learning Algorithms

# create indices for splitting (80% train, 20% test)
train_indices <- createDataPartition(lc_data$int_rate, p = 0.8, list = FALSE)

# create training and testing datasets
train_data <- lc_data[train_indices, ]
test_data <- lc_data[-train_indices, ]
#### Linear Regression ####

lm.fit <- lm(int_rate ~ ., data = train_data)

# make predictions on the training and testing data
lm.train_predictions <- predict(lm.fit, newdata = train_data)
lm.test_predictions <- predict(lm.fit, newdata = test_data)

# calculate Mean Squared Error (MSE) for training and testing
lm.train_mse <- mean((lm.train_predictions - train_data$int_rate)^2)
lm.test_mse <- mean((lm.test_predictions - test_data$int_rate)^2)

# calculate Root Mean Squared Error (RMSE) for training and testing
lm.train_rmse <- sqrt(lm.train_mse)
lm.test_rmse <- sqrt(lm.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
lm.train_mae <- mean(abs(lm.train_predictions - train_data$int_rate))
lm.test_mae <- mean(abs(lm.test_predictions - test_data$int_rate))

# calculate R-squared (R²) for training and testing
lm.train_r2 <- 1 - (sum((train_data$int_rate - lm.train_predictions)^2) / sum((train_data$int_rate - mean(train_data$int_rate))^2))
lm.test_r2 <- 1 - (sum((test_data$int_rate - lm.test_predictions)^2) / sum((test_data$int_rate - mean(test_data$int_rate))^2))

# display the metrics
cat("Training MSE:", lm.train_mse, "\n")
Training MSE: 10.33743 
cat("Testing MSE:", lm.test_mse, "\n")
Testing MSE: 10.76845 
cat("Training RMSE:", lm.train_rmse, "\n")
Training RMSE: 3.215187 
cat("Testing RMSE:", lm.test_rmse, "\n")
Testing RMSE: 3.281531 
cat("Training MAE:", lm.train_mae, "\n")
Training MAE: 2.560509 
cat("Testing MAE:", lm.test_mae, "\n")
Testing MAE: 2.563904 
cat("Training R-squared (R²):", lm.train_r2, "\n")
Training R-squared (R²): 0.4299502 
cat("Testing R-squared (R²):", lm.test_r2, "\n")
Testing R-squared (R²): 0.4069667 

Lasso It standardizes data automatically

lasso.predictors_train <- model.matrix(int_rate ~ ., train_data)[,-1]
lasso.target_train <- train_data$int_rate
lasso.predictors_test <- model.matrix(int_rate ~ ., test_data)[,-1]
lasso.target_test <- test_data$int_rate

lasso.fit <- glmnet(lasso.predictors_train, lasso.target_train, alpha = 1)

plot(lasso.fit, label=TRUE)


# make predictions on the training and testing data
lasso.train_predictions <- predict(lasso.fit, newdata = train_data, newx = lasso.predictors_train)
lasso.test_predictions <- predict(lasso.fit, newdata = test_data, newx = lasso.predictors_train)

# calculate Mean Squared Error (MSE) for training and testing
lasso.train_mse <- mean((lasso.train_predictions - train_data$int_rate)^2)
lasso.test_mse <- mean((lasso.test_predictions - test_data$int_rate)^2)
Warning: longer object length is not a multiple of shorter object length
# calculate Root Mean Squared Error (RMSE) for training and testing
lasso.train_rmse <- sqrt(lasso.train_mse)
lasso.test_rmse <- sqrt(lasso.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
lasso.train_mae <- mean(abs(lasso.train_predictions - train_data$int_rate))
lasso.test_mae <- mean(abs(lasso.test_predictions - test_data$int_rate))
Warning: longer object length is not a multiple of shorter object length
# calculate R-squared (R²) for training and testing
lasso.train_r2 <- 1 - (sum((train_data$int_rate - lasso.train_predictions)^2) / sum((train_data$int_rate - mean(train_data$int_rate))^2))
lasso.test_r2 <- 1 - (sum((test_data$int_rate - lasso.test_predictions)^2) / sum((test_data$int_rate - mean(test_data$int_rate))^2))
Warning: longer object length is not a multiple of shorter object length
# display the metrics
cat("Training MSE:", lasso.train_mse, "\n")
Training MSE: 11.57067 
cat("Testing MSE:", lasso.test_mse, "\n")
Testing MSE: 23.50268 
cat("Training RMSE:", lasso.train_rmse, "\n")
Training RMSE: 3.401569 
cat("Testing RMSE:", lasso.test_rmse, "\n")
Testing RMSE: 4.847956 
cat("Training MAE:", lasso.train_mae, "\n")
Training MAE: 2.721616 
cat("Testing MAE:", lasso.test_mae, "\n")
Testing MAE: 3.911493 
cat("Training R-squared (R²):", lasso.train_r2, "\n")
Training R-squared (R²): -46.21615 
cat("Testing R-squared (R²):", lasso.test_r2, "\n")
Testing R-squared (R²): -382.1256 

Ridge It standardizes data automatically

ridge.predictors_train <- model.matrix(int_rate ~ ., train_data)[,-1]
ridge.target_train <- train_data$int_rate
ridge.predictors_test <- model.matrix(int_rate ~ ., test_data)[,-1]
ridge.target_test <- test_data$int_rate

ridge.fit <- glmnet(ridge.predictors_train, ridge.target_train, alpha = 0)

plot(ridge.fit, label=TRUE, xlab = "L2 Norm")


# make predictions on the training and testing data
ridge.train_predictions <- predict(ridge.fit, newdata = train_data, newx = ridge.predictors_train)
ridge.test_predictions <- predict(ridge.fit, newdata = test_data, newx = ridge.predictors_train)

# calculate Mean Squared Error (MSE) for training and testing
ridge.train_mse <- mean((ridge.train_predictions - train_data$int_rate)^2)
ridge.test_mse <- mean((ridge.test_predictions - test_data$int_rate)^2)
Warning: longer object length is not a multiple of shorter object length
# calculate Root Mean Squared Error (RMSE) for training and testing
ridge.train_rmse <- sqrt(ridge.train_mse)
ridge.test_rmse <- sqrt(ridge.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
ridge.train_mae <- mean(abs(ridge.train_predictions - train_data$int_rate))
ridge.test_mae <- mean(abs(ridge.test_predictions - test_data$int_rate))
Warning: longer object length is not a multiple of shorter object length
# calculate R-squared (R²) for training and testing
ridge.train_r2 <- 1 - (sum((train_data$int_rate - ridge.train_predictions)^2) / sum((train_data$int_rate - mean(train_data$int_rate))^2))
ridge.test_r2 <- 1 - (sum((test_data$int_rate - ridge.test_predictions)^2) / sum((test_data$int_rate - mean(test_data$int_rate))^2))
Warning: longer object length is not a multiple of shorter object length
# display the metrics
cat("Training MSE:", ridge.train_mse, "\n")
Training MSE: 14.52168 
cat("Testing MSE:", ridge.test_mse, "\n")
Testing MSE: 20.06712 
cat("Training RMSE:", ridge.train_rmse, "\n")
Training RMSE: 3.810732 
cat("Testing RMSE:", ridge.test_rmse, "\n")
Testing RMSE: 4.479634 
cat("Training MAE:", ridge.train_mae, "\n")
Training MAE: 3.053025 
cat("Testing MAE:", ridge.test_mae, "\n")
Testing MAE: 3.612791 
cat("Training R-squared (R²):", ridge.train_r2, "\n")
Training R-squared (R²): -79.07873 
cat("Testing R-squared (R²):", ridge.test_r2, "\n")
Testing R-squared (R²): -441.0558 

K fold using K=5:

# define the number of folds for cross-validation
num_folds <- 5
folds <- createFolds(train_data$int_rate, k = num_folds, list = TRUE)

K fold using K=5 and linear regression:

#### Linear Regresion applying Cross Validation with k=5  ####

# initialize lists to store models and their results
lm.k5.models <- list()
lm.k5.results <- data.frame()

# perform k-fold cross-validation
for(i in seq_along(folds)) {
  # split the data into training and testing for the current fold
  train_indices <- folds[[i]]
  test_indices <- setdiff(seq_len(nrow(train_data)), train_indices)
  
  train_data_fold <- train_data[train_indices, ]
  test_data_fold <- train_data[test_indices, ]
  
  # fit the model on the training fold
  lm.k5 <- lm(int_rate ~ ., data = train_data_fold)
  lm.k5.models[[i]] <- lm.k5  # Store the model
  
  # make predictions on the training and testing fold
  lm.k5.train_predictions <- predict(lm.k5, newdata = train_data_fold)
  lm.k5.test_predictions <- predict(lm.k5, newdata = test_data_fold)
  
  # calculate metrics for training fold
  lm.k5.train_mse <- mean((lm.k5.train_predictions - train_data_fold$int_rate)^2)
  lm.k5.train_rmse <- sqrt(lm.k5.train_mse)
  lm.k5.train_mae <- mean(abs(lm.k5.train_predictions - train_data_fold$int_rate))
  lm.k5.train_r2 <- summary(lm.k5)$r.squared
  
  # calculate metrics for testing fold
  lm.k5.test_mse <- mean((lm.k5.test_predictions - test_data_fold$int_rate)^2)
  lm.k5.test_rmse <- sqrt(lm.k5.test_mse)
  lm.k5.test_mae <- mean(abs(lm.k5.test_predictions - test_data_fold$int_rate))
  lm.k5.test_r2 <- 1 - (sum((test_data_fold$int_rate - lm.k5.test_predictions)^2) / sum((test_data_fold$int_rate - mean(test_data_fold$int_rate))^2))
  
  # store metrics in the results dataframe
  lm.k5.results <- rbind(lm.k5.results, data.frame(
    Fold = i,
    Train_MSE = lm.k5.train_mse, Test_MSE = lm.k5.test_mse,
    Train_RMSE = lm.k5.train_rmse, Test_RMSE = lm.k5.test_rmse,
    Train_MAE = lm.k5.train_mae, Test_MAE = lm.k5.test_mae,
    Train_R2 = lm.k5.train_r2, Test_R2 = lm.k5.test_r2
  ))
}

# display the models and their metrics
print(lm.k5.models)
[[1]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      1.687e+00                        2.765e-05                        3.820e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.312e-02                        2.663e-01                       -6.405e-06  
            verification_status                          purpose                       addr_state  
                      7.579e-01                        3.114e-01                        7.812e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      3.244e-03                        2.937e-02                        1.833e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.535e-01                        7.908e-02                        4.089e-01  
                      revol_bal                       revol_util                        total_acc  
                      6.745e-06                        4.683e-02                       -3.022e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -9.840e-01                        3.447e-01                       -1.505e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                     -5.085e-02                        6.851e-01                        1.370e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      4.343e-05                       -1.083e-06                        6.276e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.738e-01                        6.270e-01                        1.428e-01  
                   total_bal_il                          il_util                      open_rv_12m  
                      4.173e-06                        7.194e-03                        1.974e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      3.619e-02                       -6.566e-05                       -2.253e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.496e-05                       -3.121e-02                       -3.205e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      1.041e-01                       -1.892e-01                       -1.256e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      3.029e-01                       -1.474e-01  


[[2]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.496e+00                        3.260e-05                        3.738e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.790e-02                        2.413e-01                       -3.583e-06  
            verification_status                          purpose                       addr_state  
                      6.970e-01                        3.238e-01                        2.388e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      5.065e-02                        4.697e-02                        1.832e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.412e-01                        6.030e-02                        3.292e-01  
                      revol_bal                       revol_util                        total_acc  
                      2.945e-06                        4.295e-02                       -3.410e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.026e+00                        3.403e-01                       -1.607e-05  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      5.920e-02                        4.294e-01                        1.259e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.413e-05                       -1.093e-06                       -2.077e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.338e-01                        7.100e-01                        1.539e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      3.967e-06                        3.744e-03                        1.603e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      4.186e-02                       -6.518e-05                       -3.778e-04  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.505e-05                        2.988e-02                       -4.994e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      9.086e-02                       -1.958e-01                       -2.155e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.527e-01                       -1.356e-01  


[[3]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.289e+00                        2.713e-05                        3.754e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.088e-02                        2.510e-01                       -5.645e-06  
            verification_status                          purpose                       addr_state  
                      7.132e-01                        3.364e-01                       -5.472e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.722e-02                        2.672e-02                        1.911e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.720e-01                        5.061e-02                        3.960e-01  
                      revol_bal                       revol_util                        total_acc  
                     -3.466e-06                        4.658e-02                       -3.297e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.054e+00                        3.183e-01                       -1.479e-05  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      5.008e-02                        2.727e-01                        1.299e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.562e-05                       -1.092e-06                        4.777e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.469e-01                        7.745e-01                       -2.142e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      3.063e-07                        5.769e-03                        1.045e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      8.413e-02                       -4.547e-05                       -3.304e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -4.083e-06                       -1.724e-02                       -2.834e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      9.800e-02                       -2.113e-01                       -2.008e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.711e-01                       -1.203e-01  


[[4]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      1.366e+00                        2.851e-05                        3.795e+00  
                     emp_length                   home_ownership                       annual_inc  
                      2.084e-02                        2.826e-01                       -5.818e-06  
            verification_status                          purpose                       addr_state  
                      7.511e-01                        3.187e-01                        1.663e-03  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.008e-03                        5.161e-02                        1.823e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.358e-01                        8.252e-02                        4.214e-01  
                      revol_bal                       revol_util                        total_acc  
                      7.686e-06                        4.530e-02                       -2.953e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -9.783e-01                        3.079e-01                       -2.172e-05  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      2.041e-02                        1.316e+00                        1.066e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      1.879e-05                       -1.163e-06                       -6.310e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.750e-01                        8.969e-01                       -1.558e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      4.716e-06                        3.640e-03                        2.315e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      3.048e-02                       -3.883e-05                       -4.492e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.707e-05                        1.089e-01                       -6.896e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      6.442e-02                       -1.882e-01                       -1.144e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.220e-01                       -1.363e-01  


[[5]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.001e+00                        2.174e-05                        3.796e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.740e-02                        2.294e-01                       -3.435e-06  
            verification_status                          purpose                       addr_state  
                      7.323e-01                        3.353e-01                       -3.753e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.920e-02                        3.258e-02                        1.922e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.635e-01                        4.787e-02                        4.137e-01  
                      revol_bal                       revol_util                        total_acc  
                     -5.206e-06                        4.752e-02                       -3.152e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.013e+00                        2.512e-01                       -2.362e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      4.135e-02                       -1.095e-01                        1.309e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      3.261e-05                       -1.700e-06                       -7.970e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.165e-01                        7.144e-01                        1.163e-01  
                   total_bal_il                          il_util                      open_rv_12m  
                      1.046e-06                        5.671e-03                        3.601e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      1.314e-02                       -4.553e-05                       -4.432e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -3.388e-06                        6.541e-02                       -3.427e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      3.553e-02                       -2.122e-01                       -1.419e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      3.005e-01                       -1.368e-01  
print(lm.k5.results)
plot_metric <- function(results_long, metric) {
    # adjust the variable names based on the metric
    variables <- if (metric == "OOB") {
        "OOB_Error"
    } else {
        c(paste0('Train_', metric), paste0('Test_', metric))
    }
    title <- if (metric == "OOB") {
        paste0(metric, ' per Fold')
    } else {
        paste0('Train vs Test ', metric, ' per Fold')
    }
    
    ggplot(results_long[results_long$variable %in% variables, ],
           aes(x = Fold, y = value, color = variable)) +
    geom_line() +
    geom_point() +
    theme_minimal() +
    labs(title = title,
         x = 'Fold',
         y = metric)
}
# reshape data for plotting
lm.k5.results_long <- melt(lm.k5.results, id.vars = 'Fold')

# plot for each metric
p1 <- plot_metric(lm.k5.results_long, 'MSE')
p2 <- plot_metric(lm.k5.results_long, 'RMSE')
p3 <- plot_metric(lm.k5.results_long, 'MAE')
p4 <- plot_metric(lm.k5.results_long, 'R2')

# arrange the plots in a 2x2 grid
grid.arrange(p1, p2, p3, p4, ncol = 2, nrow = 2)


plot(p1)

plot(p2)

plot(p3)

plot(p4)

K fold using K=5 and Random Forest:

#### Random Forest applying Cross Validation with k=5  ####

# initialize lists to store models and their results
rf.k5.models <- list()
rf.k5.results <- data.frame()

# perform k-fold cross-validation
for(i in seq_along(folds)) {
  # split the data into training and testing for the current fold
  train_indices <- folds[[i]]
  test_indices <- setdiff(seq_len(nrow(train_data)), train_indices)
  
  train_data_fold <- train_data[train_indices, ]
  test_data_fold <- train_data[test_indices, ]
  
  # fit the model on the training fold
  rf.k5 <- ranger(formula = int_rate ~ ., data = train_data_fold, num.trees = 500, verbose=TRUE, importance = "impurity", oob.error = TRUE)
  rf.k5.models[[i]] <- rf.k5  # Store the model
  
  # make predictions on the training and testing fold
  rf.k5.train_predictions <- predict(rf.k5, data = train_data_fold)$predictions
  rf.k5.test_predictions <- predict(rf.k5, data = test_data_fold)$predictions
  
  # calculate metrics for training fold
  rf.k5.train_mse <- mean((rf.k5.train_predictions - train_data_fold$int_rate)^2)
  rf.k5.train_rmse <- sqrt(rf.k5.train_mse)
  rf.k5.train_mae <- mean(abs(rf.k5.train_predictions - train_data_fold$int_rate))
  rf.k5.oob_error <- rf.k5$prediction.error
  
  # calculate metrics for testing fold
  rf.k5.test_mse <- mean((rf.k5.test_predictions - test_data_fold$int_rate)^2)
  rf.k5.test_rmse <- sqrt(rf.k5.test_mse)
  rf.k5.test_mae <- mean(abs(rf.k5.test_predictions - test_data_fold$int_rate))
  rf.k5.test_r2 <- 1 - (sum((test_data_fold$int_rate - rf.k5.test_predictions)^2) / sum((test_data_fold$int_rate - mean(test_data_fold$int_rate))^2))
  
  # store metrics in the results dataframe
  rf.k5.results <- rbind(rf.k5.results, data.frame(
    Fold = i,
    Train_MSE = rf.k5.train_mse, Test_MSE = rf.k5.test_mse,
    Train_RMSE = rf.k5.train_rmse, Test_RMSE = rf.k5.test_rmse,
    Train_MAE = rf.k5.train_mae, Test_MAE = rf.k5.test_mae,
    OOB_Error = rf.k5.oob_error
  ))
}
Growing trees.. Progress: 97%. Estimated remaining time: 0 seconds.
# display the models and their metrics
print(rf.k5.models)
[[1]]
Ranger result

Call:
 ranger(formula = int_rate ~ ., data = train_data_fold, num.trees = 500,      verbose = TRUE, importance = "impurity", oob.error = TRUE) 

Type:                             Regression 
Number of trees:                  500 
Sample size:                      126773 
Number of independent variables:  43 
Mtry:                             6 
Target node size:                 5 
Variable importance mode:         impurity 
Splitrule:                        variance 
OOB prediction error (MSE):       8.681077 
R squared (OOB):                  0.52044 

[[2]]
Ranger result

Call:
 ranger(formula = int_rate ~ ., data = train_data_fold, num.trees = 500,      verbose = TRUE, importance = "impurity", oob.error = TRUE) 

Type:                             Regression 
Number of trees:                  500 
Sample size:                      126774 
Number of independent variables:  43 
Mtry:                             6 
Target node size:                 5 
Variable importance mode:         impurity 
Splitrule:                        variance 
OOB prediction error (MSE):       8.752434 
R squared (OOB):                  0.5164565 

[[3]]
Ranger result

Call:
 ranger(formula = int_rate ~ ., data = train_data_fold, num.trees = 500,      verbose = TRUE, importance = "impurity", oob.error = TRUE) 

Type:                             Regression 
Number of trees:                  500 
Sample size:                      126773 
Number of independent variables:  43 
Mtry:                             6 
Target node size:                 5 
Variable importance mode:         impurity 
Splitrule:                        variance 
OOB prediction error (MSE):       8.719473 
R squared (OOB):                  0.518891 

[[4]]
Ranger result

Call:
 ranger(formula = int_rate ~ ., data = train_data_fold, num.trees = 500,      verbose = TRUE, importance = "impurity", oob.error = TRUE) 

Type:                             Regression 
Number of trees:                  500 
Sample size:                      126772 
Number of independent variables:  43 
Mtry:                             6 
Target node size:                 5 
Variable importance mode:         impurity 
Splitrule:                        variance 
OOB prediction error (MSE):       8.76257 
R squared (OOB):                  0.5168671 

[[5]]
Ranger result

Call:
 ranger(formula = int_rate ~ ., data = train_data_fold, num.trees = 500,      verbose = TRUE, importance = "impurity", oob.error = TRUE) 

Type:                             Regression 
Number of trees:                  500 
Sample size:                      126773 
Number of independent variables:  43 
Mtry:                             6 
Target node size:                 5 
Variable importance mode:         impurity 
Splitrule:                        variance 
OOB prediction error (MSE):       8.769394 
R squared (OOB):                  0.5183893 
print(rf.k5.results)
# reshape data for plotting
rf.k5.results_long <- melt(rf.k5.results, id.vars = 'Fold')

# plot for each metric
p1 <- plot_metric(rf.k5.results_long, 'MSE')
p2 <- plot_metric(rf.k5.results_long, 'RMSE')
p3 <- plot_metric(rf.k5.results_long, 'MAE')
p4 <- plot_metric(rf.k5.results_long, 'OOB')

# arrange the plots in a 2x2 grid
grid.arrange(p1, p2, p3, p4, ncol = 2, nrow = 2)


plot(p1)

plot(p2)

plot(p3)

plot(p4)

K fold using K=5 and Boosting:

#### Boosting applying Cross Validation with k=5  ####

# initialize lists to store models and their results
xgb.k5.models <- list()
xgb.k5.results <- data.frame()

# perform k-fold cross-validation
for(i in seq_along(folds)) {
  # split the data into training and testing for the current fold
  train_indices <- folds[[i]]
  test_indices <- setdiff(seq_len(nrow(train_data)), train_indices)
  
  train_data_fold <- train_data[train_indices, ]
  test_data_fold <- train_data[test_indices, ]
  
  # prepare data for xgboost
  xgb.y_train_fold <- train_data_fold$int_rate
  xgb.X_train_fold <- as.matrix(train_data_fold[, -which(names(train_data_fold) == 'int_rate')])
  
  xgb.y_test_fold <- test_data_fold$int_rate
  xgb.X_test_fold <- as.matrix(test_data_fold[, -which(names(test_data_fold) == 'int_rate')])
  
  # fit the xgboost model on the training fold
  xgb.k5 <- xgboost(
    data = xgb.X_train_fold,
    label = xgb.y_train_fold,
    nrounds = 100,
    verbose = 0
  )
  xgb.k5.models[[i]] <- xgb.k5  # store the model
  
  # make predictions on the training fold
  xgb.k5.train_predictions <- predict(xgb.k5, newdata = xgb.X_train_fold)
  # make predictions on the testing fold
  xgb.k5.test_predictions <- predict(xgb.k5, newdata = xgb.X_test_fold)
  
  # calculate metrics for training fold
  xgb.k5.train_mse <- mean((xgb.k5.train_predictions - train_data_fold$int_rate)^2)
  xgb.k5.train_rmse <- sqrt(xgb.k5.train_mse)
  xgb.k5.train_mae <- mean(abs(xgb.k5.train_predictions - train_data_fold$int_rate))
  xgb.k5.train_r2 <- 1 - (sum((xgb.y_train_fold - xgb.k5.train_predictions)^2) / sum((xgb.y_train_fold - mean(xgb.y_train_fold))^2))

  # calculate metrics for testing fold
  xgb.k5.test_mse <- mean((xgb.k5.test_predictions - xgb.y_test_fold)^2)
  xgb.k5.test_rmse <- sqrt(xgb.k5.test_mse)
  xgb.k5.test_mae <- mean(abs(xgb.k5.test_predictions - xgb.y_test_fold))
  xgb.k5.test_r2 <- 1 - (sum((xgb.y_test_fold - xgb.k5.test_predictions)^2) / sum((xgb.y_test_fold - mean(xgb.y_test_fold))^2))  
  
  # store metrics in the results dataframe
  xgb.k5.results <- rbind(xgb.k5.results, data.frame(
    Fold = i,
    Train_MSE = xgb.k5.train_mse, Test_MSE = xgb.k5.test_mse,
    Train_RMSE = xgb.k5.train_rmse, Test_RMSE = xgb.k5.test_rmse,
    Train_MAE = xgb.k5.train_mae, Test_MAE = xgb.k5.test_mae,
    Train_R2 = xgb.k5.train_r2, Test_R2 = xgb.k5.test_r2
  ))
}

# display the models and their metrics
print(xgb.k5.models)
[[1]]
##### xgb.Booster
raw: 446 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[2]]
##### xgb.Booster
raw: 450 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[3]]
##### xgb.Booster
raw: 449.8 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[4]]
##### xgb.Booster
raw: 452.3 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[5]]
##### xgb.Booster
raw: 449.9 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:
print(xgb.k5.results)
# reshape data for plotting
xgb.k5.results_long <- melt(xgb.k5.results, id.vars = 'Fold')

# plot for each metric
p1 <- plot_metric(xgb.k5.results_long, 'MSE')
p2 <- plot_metric(xgb.k5.results_long, 'RMSE')
p3 <- plot_metric(xgb.k5.results_long, 'MAE')
p4 <- plot_metric(xgb.k5.results_long, 'R2')

# arrange the plots in a 2x2 grid
grid.arrange(p1, p2, p3, p4, ncol = 2, nrow = 2)


plot(p1)

plot(p2)

plot(p3)

plot(p4)

K fold using K=10:

# define the number of folds for cross-validation
num_folds <- 10
folds <- createFolds(train_data$int_rate, k = num_folds, list = TRUE)

K fold using K=10 and linear regression:

#### Linear Regresion applying Cross Validation with k=10  ####

# initialize lists to store models and their results
lm.k10.models <- list()
lm.k10.results <- data.frame()

# perform k-fold cross-validation
for(i in seq_along(folds)) {
  # split the data into training and testing for the current fold
  train_indices <- folds[[i]]
  test_indices <- setdiff(seq_len(nrow(train_data)), train_indices)
  
  train_data_fold <- train_data[train_indices, ]
  test_data_fold <- train_data[test_indices, ]
  
  # fit the model on the training fold
  lm.k10 <- lm(int_rate ~ ., data = train_data_fold)
  lm.k10.models[[i]] <- lm.k10  # Store the model
  
  # make predictions on the training and testing fold
  lm.k10.train_predictions <- predict(lm.k10, newdata = train_data_fold)
  lm.k10.test_predictions <- predict(lm.k10, newdata = test_data_fold)
  
  # calculate metrics for training fold
  lm.k10.train_mse <- mean((lm.k10.train_predictions - train_data_fold$int_rate)^2)
  lm.k10.train_rmse <- sqrt(lm.k10.train_mse)
  lm.k10.train_mae <- mean(abs(lm.k10.train_predictions - train_data_fold$int_rate))
  lm.k10.train_r2 <- summary(lm.k10)$r.squared
  
  # calculate metrics for testing fold
  lm.k10.test_mse <- mean((lm.k10.test_predictions - test_data_fold$int_rate)^2)
  lm.k10.test_rmse <- sqrt(lm.k10.test_mse)
  lm.k10.test_mae <- mean(abs(lm.k10.test_predictions - test_data_fold$int_rate))
  lm.k10.test_r2 <- 1 - (sum((test_data_fold$int_rate - lm.k10.test_predictions)^2) / sum((test_data_fold$int_rate - mean(test_data_fold$int_rate))^2))
  
  # store metrics in the results dataframe
  lm.k10.results <- rbind(lm.k10.results, data.frame(
    Fold = i,
    Train_MSE = lm.k10.train_mse, Test_MSE = lm.k10.test_mse,
    Train_RMSE = lm.k10.train_rmse, Test_RMSE = lm.k10.test_rmse,
    Train_MAE = lm.k10.train_mae, Test_MAE = lm.k10.test_mae,
    Train_R2 = lm.k10.train_r2, Test_R2 = lm.k10.test_r2
  ))
}

# display the models and their metrics
print(lm.k10.models)
[[1]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      1.778e+00                        2.108e-05                        3.798e+00  
                     emp_length                   home_ownership                       annual_inc  
                      6.977e-03                        2.084e-01                       -3.357e-06  
            verification_status                          purpose                       addr_state  
                      7.231e-01                        3.472e-01                        2.614e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      5.258e-02                        1.391e-02                        2.000e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.567e-01                        3.993e-02                        4.354e-01  
                      revol_bal                       revol_util                        total_acc  
                     -7.100e-06                        4.768e-02                       -2.891e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.045e+00                        3.139e-01                       -2.452e-05  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      7.753e-02                        6.308e-01                        1.057e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      1.854e-05                       -1.900e-06                        1.721e-01  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.299e-01                        8.152e-01                       -9.802e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      2.824e-06                        4.530e-03                        2.507e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                     -1.891e-02                       -6.114e-05                       -5.394e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -7.678e-07                        9.840e-02                       -7.271e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      3.369e-02                       -2.143e-01                       -1.371e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.006e-01                       -1.286e-01  


[[2]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.901e+00                        2.566e-05                        3.833e+00  
                     emp_length                   home_ownership                       annual_inc  
                      2.054e-02                        2.754e-01                       -5.634e-06  
            verification_status                          purpose                       addr_state  
                      7.527e-01                        3.146e-01                        8.603e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      2.668e-03                        3.360e-02                        1.824e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.478e-01                        8.448e-02                        4.348e-01  
                      revol_bal                       revol_util                        total_acc  
                      7.358e-06                        4.626e-02                       -3.248e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -9.449e-01                        2.852e-01                       -3.859e-05  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      2.706e-01                       -2.522e-01                        1.406e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.211e-05                       -9.783e-07                       -6.330e-03  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.926e-01                        7.235e-01                        9.869e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      5.355e-06                        5.335e-03                        1.552e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      7.564e-02                       -6.177e-05                       -4.289e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.636e-05                       -1.770e-02                       -2.800e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      4.683e-02                       -1.946e-01                       -1.121e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.278e-01                       -1.417e-01  


[[3]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      3.638e+00                        2.815e-05                        3.748e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.227e-02                        2.979e-01                       -4.727e-06  
            verification_status                          purpose                       addr_state  
                      7.383e-01                        3.214e-01                        9.522e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.631e-02                        2.049e-02                        1.742e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.674e-01                        6.375e-02                        3.885e-01  
                      revol_bal                       revol_util                        total_acc  
                      5.724e-06                        4.244e-02                       -3.406e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.041e+00                        4.686e-01                       -5.396e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      4.760e-02                       -8.511e-01                        1.516e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.244e-05                       -7.431e-07                       -1.228e-01  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.050e-01                        8.781e-01                       -1.392e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                     -1.389e-06                        4.273e-03                        2.565e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      4.627e-02                       -4.357e-05                       -1.490e-04  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.673e-05                        1.024e-01                       -7.620e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      3.836e-02                       -1.998e-01                       -1.884e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.711e-01                       -1.442e-01  


[[4]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.295e+00                        2.959e-05                        3.795e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.892e-02                        2.594e-01                       -6.514e-06  
            verification_status                          purpose                       addr_state  
                      7.459e-01                        3.133e-01                       -3.027e-05  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      1.785e-03                        2.504e-02                        1.882e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.314e-01                        8.219e-02                        3.286e-01  
                      revol_bal                       revol_util                        total_acc  
                      8.685e-06                        4.601e-02                       -2.954e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -9.616e-01                        5.323e-01                       -6.292e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                     -4.154e-02                        4.494e-01                        1.095e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.995e-05                       -1.247e-06                       -7.437e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.234e-01                        7.278e-01                        9.718e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      2.178e-06                        7.526e-03                        3.600e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      2.387e-03                       -1.382e-06                       -8.134e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.639e-05                        1.741e-01                       -6.909e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      6.917e-02                       -2.081e-01                       -2.069e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      3.135e-01                       -1.182e-01  


[[5]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.998e+00                        3.093e-05                        3.756e+00  
                     emp_length                   home_ownership                       annual_inc  
                      9.611e-03                        2.404e-01                       -2.415e-06  
            verification_status                          purpose                       addr_state  
                      6.917e-01                        3.243e-01                        2.593e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      5.377e-02                        1.772e-02                        1.827e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.168e-01                        6.338e-02                        4.192e-01  
                      revol_bal                       revol_util                        total_acc  
                      4.110e-06                        4.327e-02                       -3.438e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.002e+00                        1.247e-01                        6.189e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                     -6.133e-02                       -2.462e-01                        7.604e-01  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      4.426e-05                       -1.149e-06                        6.598e-03  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.448e-01                        3.746e-01                        1.124e-01  
                   total_bal_il                          il_util                      open_rv_12m  
                      3.174e-07                        5.593e-03                        1.588e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      2.754e-02                       -6.245e-05                       -2.970e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.691e-05                       -5.206e-02                       -1.046e-01  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      9.716e-02                       -2.083e-01                       -9.049e-02  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      1.455e-01                       -1.310e-01  


[[6]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.067e-02                        3.412e-05                        3.694e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.364e-02                        2.492e-01                       -6.128e-06  
            verification_status                          purpose                       addr_state  
                      6.959e-01                        3.290e-01                       -2.169e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.054e-02                        5.693e-02                        1.741e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.554e-01                        7.110e-02                        4.577e-01  
                      revol_bal                       revol_util                        total_acc  
                      7.136e-06                        4.279e-02                       -3.529e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.014e+00                        2.466e-01                        9.446e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                     -3.202e-01                        2.229e+00                        1.390e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.225e-05                       -8.418e-07                        7.038e-03  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -2.195e-01                        6.524e-01                        1.341e-01  
                   total_bal_il                          il_util                      open_rv_12m  
                      3.279e-06                        4.687e-03                        2.853e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      6.596e-03                       -6.769e-05                        4.724e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.697e-05                       -4.337e-02                       -4.166e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      1.036e-01                       -1.706e-01                       -1.246e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      3.146e-01                       -1.415e-01  


[[7]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      1.825e+00                        3.079e-05                        3.741e+00  
                     emp_length                   home_ownership                       annual_inc  
                      2.133e-02                        2.608e-01                       -4.716e-06  
            verification_status                          purpose                       addr_state  
                      7.121e-01                        3.155e-01                        4.318e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.771e-02                        4.869e-02                        1.835e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.860e-01                        6.803e-02                        3.876e-01  
                      revol_bal                       revol_util                        total_acc  
                      6.986e-06                        4.247e-02                       -3.654e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.023e+00                        1.999e-01                       -1.387e-05  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      2.978e-02                        4.312e-01                        1.502e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      4.461e-05                       -1.036e-06                        1.088e-01  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.352e-01                        7.271e-01                        6.745e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      2.335e-06                        3.981e-03                        1.189e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      8.138e-02                       -6.028e-05                       -1.924e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.640e-05                        1.844e-02                       -8.906e-03  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      7.226e-02                       -1.914e-01                       -1.429e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.999e-01                       -1.458e-01  


[[8]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.037e+00                        2.574e-05                        3.798e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.503e-02                        2.297e-01                       -4.558e-06  
            verification_status                          purpose                       addr_state  
                      7.240e-01                        3.290e-01                       -7.392e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.611e-02                        6.752e-02                        1.974e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.740e-01                        3.992e-02                        4.222e-01  
                      revol_bal                       revol_util                        total_acc  
                     -9.588e-06                        4.823e-02                       -2.884e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -1.051e+00                        3.011e-01                       -4.061e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      3.964e-03                       -6.782e-02                        1.081e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.240e-05                       -1.498e-06                       -3.156e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.191e-01                        6.820e-01                        4.732e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      3.521e-06                        4.768e-03                        1.401e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      9.578e-02                       -6.459e-05                       -2.557e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                      5.959e-08                        6.987e-02                       -2.691e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      9.257e-02                       -1.942e-01                       -1.746e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      3.119e-01                       -1.327e-01  


[[9]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      2.355e+00                        3.492e-05                        3.748e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.737e-02                        2.636e-01                       -5.943e-06  
            verification_status                          purpose                       addr_state  
                      7.067e-01                        3.203e-01                        6.919e-05  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.886e-02                        6.105e-02                        1.851e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.845e-01                        5.654e-02                        3.315e-01  
                      revol_bal                       revol_util                        total_acc  
                      2.492e-06                        4.419e-02                       -3.310e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -9.784e-01                        2.698e-01                        7.044e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                      4.701e-02                       -3.011e-01                        1.528e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      3.814e-05                       -8.883e-07                       -1.165e-01  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.696e-01                        9.205e-01                        6.127e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      4.596e-06                        6.000e-03                        2.665e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      6.598e-02                       -1.976e-05                       -5.193e-03  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.410e-05                       -7.860e-02                       -3.716e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      1.172e-01                       -1.987e-01                       -1.974e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      3.706e-01                       -1.402e-01  


[[10]]

Call:
lm(formula = int_rate ~ ., data = train_data_fold)

Coefficients:
                    (Intercept)                        loan_amnt                             term  
                      1.451e+00                        3.071e-05                        3.727e+00  
                     emp_length                   home_ownership                       annual_inc  
                      1.764e-02                        2.607e-01                       -4.339e-06  
            verification_status                          purpose                       addr_state  
                      7.309e-01                        3.389e-01                        5.991e-04  
                            dti                      delinq_2yrs                 earliest_cr_line  
                      4.680e-02                        2.040e-02                        1.715e-09  
                 inq_last_6mths                         open_acc                          pub_rec  
                      9.219e-01                        6.120e-02                        3.462e-01  
                      revol_bal                       revol_util                        total_acc  
                      8.017e-06                        4.183e-02                       -3.057e-02  
            initial_list_status       collections_12_mths_ex_med                 annual_inc_joint  
                     -9.882e-01                        4.758e-01                       -7.030e-06  
                      dti_joint        verification_status_joint                   acc_now_delinq  
                     -5.550e-02                        1.305e+00                        1.684e+00  
                   tot_coll_amt                      tot_cur_bal                      open_acc_6m  
                      2.780e-05                       -1.084e-06                        1.940e-02  
                     open_il_6m                      open_il_12m                      open_il_24m  
                     -1.582e-01                        8.023e-01                       -2.529e-02  
                   total_bal_il                          il_util                      open_rv_12m  
                      2.130e-06                        2.553e-03                        2.229e-01  
                    open_rv_24m                       max_bal_bc                         all_util  
                      7.609e-03                       -6.233e-05                        5.916e-04  
               total_rev_hi_lim                           inq_fi                      total_cu_tl  
                     -1.714e-05                        5.684e-02                        1.032e-02  
                   inq_last_12m            mths_since_delinq_cat       mths_since_last_record_cat  
                      9.648e-02                       -2.010e-01                       -2.017e-01  
         mths_since_rcnt_il_cat  mths_since_last_major_derog_cat  
                      2.745e-01                       -1.373e-01  
print(lm.k10.results)
plot_metric <- function(results_long, metric) {
    # adjust the variable names based on the metric
    variables <- if (metric == "OOB") {
        "OOB_Error"
    } else {
        c(paste0('Train_', metric), paste0('Test_', metric))
    }
    title <- if (metric == "OOB") {
        paste0(metric, ' per Fold')
    } else {
        paste0('Train vs Test ', metric, ' per Fold')
    }
    
    ggplot(results_long[results_long$variable %in% variables, ],
           aes(x = Fold, y = value, color = variable)) +
    geom_line() +
    geom_point() +
    theme_minimal() +
    labs(title = title,
         x = 'Fold',
         y = metric)
}
# reshape data for plotting
lm.k10.results_long <- melt(lm.k10.results, id.vars = 'Fold')

# plot for each metric
p1 <- plot_metric(lm.k10.results_long, 'MSE')
p2 <- plot_metric(lm.k10.results_long, 'RMSE')
p3 <- plot_metric(lm.k10.results_long, 'MAE')
p4 <- plot_metric(lm.k10.results_long, 'R2')

# arrange the plots in a 2x2 grid
grid.arrange(p1, p2, p3, p4, ncol = 2, nrow = 2)


plot(p1)

plot(p2)

plot(p3)

plot(p4)

K fold using K=10 and Random Forest:

# #### Random Forest applying Cross Validation with k=10  ####
# 
# # initialize lists to store models and their results
# rf.k10.models <- list()
# rf.k10.results <- data.frame()
# 
# # perform k-fold cross-validation
# for(i in seq_along(folds)) {
#   # split the data into training and testing for the current fold
#   train_indices <- folds[[i]]
#   test_indices <- setdiff(seq_len(nrow(train_data)), train_indices)
#   
#   train_data_fold <- train_data[train_indices, ]
#   test_data_fold <- train_data[test_indices, ]
#   
#   # fit the model on the training fold
#   rf.k10 <- ranger(formula = int_rate ~ ., data = train_data, num.trees = 500, verbose=TRUE, importance = "impurity", oob.error = TRUE)
#   rf.k10.models[[i]] <- rf.k10  # Store the model
#   
#   # make predictions on the training and testing fold
#   rf.k10.train_predictions <- predict(rf.k10, data = train_data_fold)$predictions
#   rf.k10.test_predictions <- predict(rf.k10, data = test_data_fold)$predictions
#   
#   # calculate metrics for training fold
#   rf.k10.train_mse <- mean((rf.k10.train_predictions - train_data_fold$int_rate)^2)
#   rf.k10.train_rmse <- sqrt(rf.k10.train_mse)
#   rf.k10.train_mae <- mean(abs(rf.k10.train_predictions - train_data_fold$int_rate))
#   rf.k10.oob_error <- rf.k10$prediction.error
#   
#   # calculate metrics for testing fold
#   rf.k10.test_mse <- mean((rf.k10.test_predictions - test_data_fold$int_rate)^2)
#   rf.k10.test_rmse <- sqrt(rf.k10.test_mse)
#   rf.k10.test_mae <- mean(abs(rf.k10.test_predictions - test_data_fold$int_rate))
#   rf.k10.test_r2 <- 1 - (sum((test_data_fold$int_rate - rf.k10.test_predictions)^2) / sum((test_data_fold$int_rate - mean(test_data_fold$int_rate))^2))
#   
#   # store metrics in the results dataframe
#   rf.k10.results <- rbind(rf.k10.results, data.frame(
#     Fold = i,
#     Train_MSE = rf.k10.train_mse, Test_MSE = rf.k10.test_mse,
#     Train_RMSE = rf.k10.train_rmse, Test_RMSE = rf.k10.test_rmse,
#     Train_MAE = rf.k10.train_mae, Test_MAE = rf.k10.test_mae,
#     OOB_Error = rf.k10.oob_error
#   ))
# }
# 
# # display the models and their metrics
# print(rf.k10.models)
# print(rf.k10.results)
# reshape data for plotting
# rf.k10.results_long <- melt(rf.k10.results, id.vars = 'Fold')
# 
# # plot for each metric
# p1 <- plot_metric(rf.k10.results_long, 'MSE')
# p2 <- plot_metric(rf.k10.results_long, 'RMSE')
# p3 <- plot_metric(rf.k10.results_long, 'MAE')
# p4 <- plot_metric(rf.k10.results_long, 'OOB')
# 
# # arrange the plots in a 2x2 grid
# grid.arrange(p1, p2, p3, p4, ncol = 2, nrow = 2)
# 
# plot(p1)
# plot(p2)
# plot(p3)
# plot(p4)

K fold using K=10 and Boosting:

#### Boosting applying Cross Validation with k=10  ####

# initialize lists to store models and their results
xgb.k10.models <- list()
xgb.k10.results <- data.frame()

# perform k-fold cross-validation
for(i in seq_along(folds)) {
  # split the data into training and testing for the current fold
  train_indices <- folds[[i]]
  test_indices <- setdiff(seq_len(nrow(train_data)), train_indices)
  
  train_data_fold <- train_data[train_indices, ]
  test_data_fold <- train_data[test_indices, ]
  
  # prepare data for xgboost
  xgb.y_train_fold <- train_data_fold$int_rate
  xgb.X_train_fold <- as.matrix(train_data_fold[, -which(names(train_data_fold) == 'int_rate')])
  
  xgb.y_test_fold <- test_data_fold$int_rate
  xgb.X_test_fold <- as.matrix(test_data_fold[, -which(names(test_data_fold) == 'int_rate')])
  
  # fit the xgboost model on the training fold
  xgb.k10 <- xgboost(
    data = xgb.X_train_fold,
    label = xgb.y_train_fold,
    nrounds = 100,
    verbose = 0
  )
  xgb.k10.models[[i]] <- xgb.k10  # store the model
  
  # make predictions on the training fold
  xgb.k10.train_predictions <- predict(xgb.k10, newdata = xgb.X_train_fold)
  # make predictions on the testing fold
  xgb.k10.test_predictions <- predict(xgb.k10, newdata = xgb.X_test_fold)
  
  # calculate metrics for training fold
  xgb.k10.train_mse <- mean((xgb.k10.train_predictions - train_data_fold$int_rate)^2)
  xgb.k10.train_rmse <- sqrt(xgb.k10.train_mse)
  xgb.k10.train_mae <- mean(abs(xgb.k10.train_predictions - train_data_fold$int_rate))
  xgb.k10.train_r2 <- 1 - (sum((xgb.y_train_fold - xgb.k10.train_predictions)^2) / sum((xgb.y_train_fold - mean(xgb.y_train_fold))^2))

  # calculate metrics for testing fold
  xgb.k10.test_mse <- mean((xgb.k10.test_predictions - xgb.y_test_fold)^2)
  xgb.k10.test_rmse <- sqrt(xgb.k10.test_mse)
  xgb.k10.test_mae <- mean(abs(xgb.k10.test_predictions - xgb.y_test_fold))
  xgb.k10.test_r2 <- 1 - (sum((xgb.y_test_fold - xgb.k10.test_predictions)^2) / sum((xgb.y_test_fold - mean(xgb.y_test_fold))^2))  
  
  # store metrics in the results dataframe
  xgb.k10.results <- rbind(xgb.k10.results, data.frame(
    Fold = i,
    Train_MSE = xgb.k10.train_mse, Test_MSE = xgb.k10.test_mse,
    Train_RMSE = xgb.k10.train_rmse, Test_RMSE = xgb.k10.test_rmse,
    Train_MAE = xgb.k10.train_mae, Test_MAE = xgb.k10.test_mae,
    Train_R2 = xgb.k10.train_r2, Test_R2 = xgb.k10.test_r2
  ))
}

# display the models and their metrics
print(xgb.k10.models)
[[1]]
##### xgb.Booster
raw: 435.5 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[2]]
##### xgb.Booster
raw: 430.4 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[3]]
##### xgb.Booster
raw: 434 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[4]]
##### xgb.Booster
raw: 436 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[5]]
##### xgb.Booster
raw: 445.6 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[6]]
##### xgb.Booster
raw: 432.4 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[7]]
##### xgb.Booster
raw: 433.6 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[8]]
##### xgb.Booster
raw: 442.7 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[9]]
##### xgb.Booster
raw: 438.7 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:

[[10]]
##### xgb.Booster
raw: 448.8 Kb 
call:
  xgb.train(params = params, data = dtrain, nrounds = nrounds, 
    watchlist = watchlist, verbose = verbose, print_every_n = print_every_n, 
    early_stopping_rounds = early_stopping_rounds, maximize = maximize, 
    save_period = save_period, save_name = save_name, xgb_model = xgb_model, 
    callbacks = callbacks)
params (as set within xgb.train):
  validate_parameters = "TRUE"
xgb.attributes:
  niter
callbacks:
  cb.evaluation.log()
# of features: 43 
niter: 100
nfeatures : 43 
evaluation_log:
print(xgb.k10.results)
# reshape data for plotting
xgb.k10.results_long <- melt(xgb.k10.results, id.vars = 'Fold')

# plot for each metric
p1 <- plot_metric(xgb.k10.results_long, 'MSE')
p2 <- plot_metric(xgb.k10.results_long, 'RMSE')
p3 <- plot_metric(xgb.k10.results_long, 'MAE')
p4 <- plot_metric(xgb.k10.results_long, 'R2')

# arrange the plots in a 2x2 grid
grid.arrange(p1, p2, p3, p4, ncol = 2, nrow = 2)


plot(p1)

plot(p2)

plot(p3)

plot(p4)

Decision Trees

#### Decision Trees ####

# error in tree: "factor predictors must have at most 32 levels" is thrown

# basically, it becomes computationally expensive to create so many splits in your data, since you are selecting the best split out of all 2^32 (approx) possible splits

# The error above was solved with the factor and then numeric variable transformation

# fit a decision tree model on the training data
tm <- tree(int_rate ~ ., data = train_data)

# make predictions on the training and testing data
tm.train_predictions <- predict(tm, newdata = train_data)
tm.test_predictions <- predict(tm, newdata = test_data)

# calculate Mean Squared Error (MSE) for training and testing
tm.train_mse <- mean((tm.train_predictions - train_data$int_rate)^2)
tm.test_mse <- mean((tm.test_predictions - test_data$int_rate)^2)

# calculate Root Mean Squared Error (RMSE) for training and testing
tm.train_rmse <- sqrt(tm.train_mse)
tm.test_rmse <- sqrt(tm.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
tm.train_mae <- mean(abs(tm.train_predictions - train_data$int_rate))
tm.test_mae <- mean(abs(tm.test_predictions - test_data$int_rate))

# calculate R-squared (R²) for training and testing
tm.train_r2 <- 1 - (sum((train_data$int_rate - tm.train_predictions)^2) / sum((train_data$int_rate - mean(train_data$int_rate))^2))
tm.test_r2 <- 1 - (sum((test_data$int_rate - tm.test_predictions)^2) / sum((test_data$int_rate - mean(test_data$int_rate))^2))

# display the metrics
cat("Training MSE:", tm.train_mse, "\n")
Training MSE: 13.38487 
cat("Testing MSE:", tm.test_mse, "\n")
Testing MSE: 13.4105 
cat("Training RMSE:", tm.train_rmse, "\n")
Training RMSE: 3.658534 
cat("Testing RMSE:", tm.test_rmse, "\n")
Testing RMSE: 3.662034 
cat("Training MAE:", tm.train_mae, "\n")
Training MAE: 2.953433 
cat("Testing MAE:", tm.test_mae, "\n")
Testing MAE: 2.953717 
cat("Training R-squared (R²):", tm.train_r2, "\n")
Training R-squared (R²): 0.2619011 
cat("Testing R-squared (R²):", tm.test_r2, "\n")
Testing R-squared (R²): 0.2614654 

Random Forest

#### Random Forest ####

# train a Random Forest model
rf <- ranger(formula = int_rate ~ ., data = train_data, num.trees = 500, verbose=TRUE, importance = "impurity", oob.error = TRUE)
Growing trees.. Progress: 15%. Estimated remaining time: 2 minutes, 58 seconds.
Growing trees.. Progress: 31%. Estimated remaining time: 2 minutes, 21 seconds.
Growing trees.. Progress: 47%. Estimated remaining time: 1 minute, 47 seconds.
Growing trees.. Progress: 62%. Estimated remaining time: 1 minute, 15 seconds.
Growing trees.. Progress: 78%. Estimated remaining time: 44 seconds.
Growing trees.. Progress: 93%. Estimated remaining time: 13 seconds.

Boosting

#### Boosting ####

# define the target variable for training and testing
xgb.y_train <- train_data$int_rate
xgb.y_test <- test_data$int_rate

# define the feature matrix for training and testing (exclude the target variable)
xgb.X_train <- train_data[, -which(names(train_data) == 'int_rate')]
xgb.X_test <- test_data[, -which(names(test_data) == 'int_rate')]

# fit a gradient boosting regression model using xgboost
xgb <- xgboost(
  data = as.matrix(xgb.X_train),
  label = xgb.y_train,
  nrounds = 100,
  verbose = 0
)

# make predictions on the training and testing data
xgb.train_predictions <- predict(xgb, newdata = as.matrix(xgb.X_train))
xgb.test_predictions <- predict(xgb, newdata = as.matrix(xgb.X_test))

# calculate Mean Squared Error (MSE) for training and testing
xgb.train_mse <- mean((xgb.train_predictions - xgb.y_train)^2)
xgb.test_mse <- mean((xgb.test_predictions - xgb.y_test)^2)

# calculate Root Mean Squared Error (RMSE) for training and testing
xgb.train_rmse <- sqrt(xgb.train_mse)
xgb.test_rmse <- sqrt(xgb.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
xgb.train_mae <- mean(abs(xgb.train_predictions - xgb.y_train))
xgb.test_mae <- mean(abs(xgb.test_predictions - xgb.y_test))

# calculate R-squared (R²) for training and testing
xgb.train_r2 <- 1 - (sum((xgb.y_train - xgb.train_predictions)^2) / sum((xgb.y_train - mean(xgb.y_train))^2))
xgb.test_r2 <- 1 - (sum((xgb.y_test - xgb.test_predictions)^2) / sum((xgb.y_test - mean(xgb.y_test))^2))

# display the metrics
cat("Training MSE:", xgb.train_mse, "\n")
Training MSE: 7.430606 
cat("Testing MSE:", xgb.test_mse, "\n")
Testing MSE: 7.697747 
cat("Training RMSE:", xgb.train_rmse, "\n")
Training RMSE: 2.725914 
cat("Testing RMSE:", xgb.test_rmse, "\n")
Testing RMSE: 2.774481 
cat("Training MAE:", xgb.train_mae, "\n")
Training MAE: 2.147623 
cat("Testing MAE:", xgb.test_mae, "\n")
Testing MAE: 2.185803 
cat("Training R-squared (R²):", xgb.train_r2, "\n")
Training R-squared (R²): 0.5902447 
cat("Testing R-squared (R²):", xgb.test_r2, "\n")
Testing R-squared (R²): 0.5760744 

Following, a scatter plot of actual vs predicted training values for each model is plot. This plot helps us visualize how well each model’s predictions align with the actual data points.

# create a scatter plot function
create_scatter_plot <- function(actual_values, predicted_values, model_name) {
  model_comparison_data <- data.frame(
    Actual = actual_values,
    Predicted = predicted_values
  )
  
  scatter_plot <- ggplot(model_comparison_data, aes(x = Actual, y = Predicted)) +
    geom_point() +
    geom_abline(intercept = 0, slope = 1, linetype = "dashed", color = "red") +  # add a diagonal reference line
    labs(x = "Actual Training Values", y = "Predicted Training Values", title = model_name) +
    theme_minimal() +
    ylim(-50, 50)
  
  return(scatter_plot)
}

# create scatter plots for each model
lm_scatter_plot <- create_scatter_plot(
  actual_values = train_data$int_rate,
  predicted_values = lm.train_predictions,
  model_name = "Linear Regression"
)

rf_scatter_plot <- create_scatter_plot(
  actual_values = train_data$int_rate,
  predicted_values = rf.train_predictions$predictions,
  model_name = "Random Forest"
)

xgb_scatter_plot <- create_scatter_plot(
  actual_values = xgb.y_train,
  predicted_values = xgb.train_predictions,
  model_name = "XGBoost"
)

# display the scatter plots separately
print(lm_scatter_plot)

print(rf_scatter_plot)

print(xgb_scatter_plot)

Following, a scatter plot of actual vs predicted testing values for each model is plot. This plot helps us visualize how well each model’s predictions align with the actual data points.

# create a scatter plot function
create_scatter_plot <- function(actual_values, predicted_values, model_name) {
  model_comparison_data <- data.frame(
    Actual = actual_values,
    Predicted = predicted_values
  )
  
  scatter_plot <- ggplot(model_comparison_data, aes(x = Actual, y = Predicted)) +
    geom_point() +
    geom_abline(intercept = 0, slope = 1, linetype = "dashed", color = "red") +  # add a diagonal reference line
    labs(x = "Actual Testing Values", y = "Predicted Testing Values", title = model_name) +
    theme_minimal() +
    ylim(-50, 50) +
    xlim(0, 40)
  
  return(scatter_plot)
}

# create scatter plots for each model
lm_scatter_plot <- create_scatter_plot(
  actual_values = test_data$int_rate,
  predicted_values = lm.test_predictions,
  model_name = "Linear Regression"
)

rf_scatter_plot <- create_scatter_plot(
  actual_values = test_data$int_rate,
  predicted_values = rf.test_predictions$predictions,
  model_name = "Random Forest"
)

xgb_scatter_plot <- create_scatter_plot(
  actual_values = xgb.y_test,
  predicted_values = xgb.test_predictions,
  model_name = "XGBoost"
)

# display the scatter plots separately
print(lm_scatter_plot)

print(rf_scatter_plot)

print(xgb_scatter_plot)

Residual plots can help identify patterns in prediction errors and assess whether the assumptions of linear regression (if applicable) are met.

# create a residual plot function
create_residual_plot <- function(actual_values, predicted_values, model_name) {
  residuals <- actual_values - predicted_values
  residual_data <- data.frame(
    Predicted = predicted_values,
    Residuals = residuals
  )
  
  residual_plot <- ggplot(residual_data, aes(x = Predicted, y = Residuals)) +
    geom_point() +
    geom_hline(yintercept = 0, linetype = "dashed", color = "red") +  # Red horizontal reference line
    labs(x = "Predicted Values", y = "Residuals", title = paste("Residual Plot -", model_name)) +
    theme_minimal() +
    ylim(-30, 30) +
    xlim(0, 40)
  
  return(residual_plot)
}

# create residual plots for each model
lm_residual_plot <- create_residual_plot(
  actual_values = train_data$int_rate,
  predicted_values = lm.train_predictions,
  model_name = "Linear Regression"
)

rf_residual_plot <- create_residual_plot(
  actual_values = train_data$int_rate,
  predicted_values = rf.train_predictions$predictions,
  model_name = "Random Forest"
)

xgb_residual_plot <- create_residual_plot(
  actual_values = xgb.y_train,
  predicted_values = xgb.train_predictions,
  model_name = "XGBoost"
)

# display the residual plots separately
print(lm_residual_plot)
# create a density plot function for residuals
create_residual_density_plot <- function(actual_values, predicted_values, model_name) {
  residuals <- actual_values - predicted_values
  residual_data <- data.frame(Residuals = residuals)
  
  density_plot <- ggplot(residual_data, aes(x = Residuals)) +
    geom_density(fill = "skyblue", color = "black", alpha = 0.7) +
    labs(x = "Residuals", y = "Density", title = paste("Residual Density Plot -", model_name)) +
    theme_minimal() +
    xlim(-30,30) + 
    ylim(0, 0.35)
    
  
  return(density_plot)
}

# create density plots for residuals for each model
lm_residual_density_plot <- create_residual_density_plot(
  actual_values = train_data$int_rate,
  predicted_values = lm.train_predictions,
  model_name = "Linear Regression"
)

rf_residual_density_plot <- create_residual_density_plot(
  actual_values = train_data$int_rate,
  predicted_values = rf.train_predictions$predictions,
  model_name = "Random Forest"
)

xgb_residual_density_plot <- create_residual_density_plot(
  actual_values = xgb.y_train,
  predicted_values = xgb.train_predictions,
  model_name = "XGBoost"
)

# display the density plots separately
print(lm_residual_density_plot)

print(rf_residual_density_plot)

print(xgb_residual_density_plot)

This visualization can help you compare the distribution of prediction errors across models through histograms.

# create a histogram plot function for residuals with a red density curve
create_residual_histogram_plot <- function(actual_values, predicted_values, model_name) {
  residuals <- actual_values - predicted_values
  residual_data <- data.frame(Residuals = residuals)
  
  histogram_plot <- ggplot(residual_data, aes(x = Residuals)) +
    geom_histogram(aes(y = after_stat(density)), bins = 30, fill = "skyblue", color = "black", alpha = 0.7) +  # use density on the y-axis for the histogram
    geom_density(color = "red", linewidth = 1.5) +  # add the density plot in red
    labs(x = "Residuals", y = "Density", title = paste("Residual Histogram Plot with Density Curve -", model_name)) +
    theme_minimal() +
    xlim(-20,20) + 
    ylim(0, 0.3)
  
  return(histogram_plot)
}

# create histogram plots for residuals for each model
lm_residual_histogram_plot <- create_residual_histogram_plot(
  actual_values = train_data$int_rate,
  predicted_values = lm.train_predictions,
  model_name = "Linear Regression"
)

rf_residual_histogram_plot <- create_residual_histogram_plot(
  actual_values = train_data$int_rate,
  predicted_values = rf.train_predictions$predictions,
  model_name = "Random Forest"
)

xgb_residual_histogram_plot <- create_residual_histogram_plot(
  actual_values = xgb.y_train,
  predicted_values = xgb.train_predictions,
  model_name = "XGBoost"
)

# display the histogram plots separately
print(lm_residual_histogram_plot)

print(rf_residual_histogram_plot)

print(xgb_residual_histogram_plot)

For each model a bar chart that displays the R-squared (coefficient of determination) values is created. R-squared measures the proportion of variance in the target variable explained by the model. Higher R-squared values indicate better model fit.

# create a data frame with R-squared values for each model
model_names <- c("Linear Regression", "Random Forest", "XGBoost")
r_squared_values_train <- c(
  lm.train_r2,
  rf.train_r2,
  xgb.train_r2
)
r_squared_values_test <- c(
  lm.test_r2,
  rf.test_r2,
  xgb.test_r2
)

r_squared_data_train <- data.frame(Model = factor(model_names),
                              R_squared = r_squared_values_train)
r_squared_data_test <- data.frame(Model = factor(model_names),
                              R_squared = r_squared_values_test)

# create the R-squared comparison bar chart
r_squared_bar_chart_train <- ggplot(r_squared_data_train, aes(x = Model, y = R_squared, fill = Model)) +
  geom_bar(stat = "identity") +
  labs(x = "Model", y = "R-squared (R²)", title = "R-squared Comparison Training") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) + 
  ylim(0,1)
r_squared_bar_chart_test <- ggplot(r_squared_data_test, aes(x = Model, y = R_squared, fill = Model)) +
  geom_bar(stat = "identity") +
  labs(x = "Model", y = "R-squared (R²)", title = "R-squared Comparison Testing") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) + 
  ylim(0,1)

# display the R-squared comparison bar chart
print(r_squared_bar_chart_train)

print(r_squared_bar_chart_test)

A bar chart that compares the MAE or RMSE values, is generated for each model. These metrics quantify the average prediction errors of each model, and lower values are preferred.

# create a data frame with MAE and RMSE values for each model
model_names <- c("Linear Regression", "Random Forest", "XGBoost","Linear Regression", "Random Forest", "XGBoost")
error_values_train <- c(
  lm.train_mae,
  rf.train_mae,
  xgb.train_mae,
  lm.train_rmse,
  rf.train_rmse,
  xgb.train_rmse
)
error_values_test <- c(
  lm.test_mae,
  rf.test_mae,
  xgb.test_mae,
  lm.test_rmse,
  rf.test_rmse,
  xgb.test_rmse
)
error_type <- c(
  "MAE", "MAE", "MAE","RMSE","RMSE","RMSE"
)
model_errors_train <- data.frame(Model = factor(model_names, levels = c("Linear Regression", "Random Forest", "XGBoost")),
                Error = error_values_train, Type = error_type)
model_errors_test <- data.frame(Model = factor(model_names, levels = c("Linear Regression", "Random Forest", "XGBoost")),
                Error = error_values_test, Type = error_type)
# create the MAE or RMSE comparison bar chart
error_bar_chart_train <- ggplot(model_errors_train, aes(x = Model, y = Error, fill = Type)) +
  geom_bar(stat = "identity", position = "dodge") +
  labs(x = "Model", y = "Error Value", title = "Training MAE and RMSE Comparison") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) + 
  ylim(0, 4)

error_bar_chart_test <- ggplot(model_errors_test, aes(x = Model, y = Error, fill = Type)) +
  geom_bar(stat = "identity", position = "dodge") +
  labs(x = "Model", y = "Error Value", title = "Testing MAE and RMSE Comparison") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) + 
  ylim(0, 4)

# display the MAE and RMSE comparison bar chart
print(error_bar_chart_train)

print(error_bar_chart_test)

#### Random Forest Feature Importance Plot ####
v1 <- vip(rf, title = "Ranger", num_features = 20) 
plot(v1)

Feature Selection from the variable importance’s analysis:

imp.variables <- lc_data[, v1$data$Variable]
imp.variables$int_rate <- lc_data$int_rate
imp.train_indices <- createDataPartition(imp.variables$int_rate, p = 0.8, list = FALSE)

# create training and testing datasets
imp.train_data <- imp.variables[imp.train_indices, ]
imp.test_data <- imp.variables[-imp.train_indices, ]
#### Linear Regression with only importance variables ####

imp.lm.fit <- lm(int_rate ~ ., data = imp.train_data)

# make predictions on the training and testing data
imp.lm.train_predictions <- predict(imp.lm.fit, newdata = imp.train_data)
imp.lm.test_predictions <- predict(imp.lm.fit, newdata = imp.test_data)

# calculate Mean Squared Error (MSE) for training and testing
imp.lm.train_mse <- mean((imp.lm.train_predictions - imp.train_data$int_rate)^2)
imp.lm.test_mse <- mean((imp.lm.test_predictions - imp.test_data$int_rate)^2)

# calculate Root Mean Squared Error (RMSE) for training and testing
imp.lm.train_rmse <- sqrt(imp.lm.train_mse)
imp.lm.test_rmse <- sqrt(imp.lm.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
imp.lm.train_mae <- mean(abs(imp.lm.train_predictions - imp.train_data$int_rate))
imp.lm.test_mae <- mean(abs(imp.lm.test_predictions - imp.test_data$int_rate))

# calculate R-squared (R²) for training and testing
imp.lm.train_r2 <- 1 - (sum((imp.train_data$int_rate - imp.lm.train_predictions)^2) / sum((imp.train_data$int_rate - mean(imp.train_data$int_rate))^2))
imp.lm.test_r2 <- 1 - (sum((imp.test_data$int_rate - imp.lm.test_predictions)^2) / sum((imp.test_data$int_rate - mean(imp.test_data$int_rate))^2))

# display the metrics
cat("Training MSE:", imp.lm.train_mse, "\n")
Training MSE: 10.38227 
cat("Testing MSE:", imp.lm.test_mse, "\n")
Testing MSE: 13.5138 
cat("Training RMSE:", imp.lm.train_rmse, "\n")
Training RMSE: 3.222153 
cat("Testing RMSE:", imp.lm.test_rmse, "\n")
Testing RMSE: 3.676112 
cat("Training MAE:", imp.lm.train_mae, "\n")
Training MAE: 2.57012 
cat("Testing MAE:", imp.lm.test_mae, "\n")
Testing MAE: 2.581021 
cat("Training R-squared (R²):", imp.lm.train_r2, "\n")
Training R-squared (R²): 0.4272634 
cat("Testing R-squared (R²):", imp.lm.test_r2, "\n")
Testing R-squared (R²): 0.2568865 
#### Random Forest with only importance variables ####

# train a Random Forest model
imp.rf <- ranger(formula = int_rate ~ ., data = imp.train_data, num.trees = 500, verbose=TRUE, importance = "impurity", oob.error = TRUE)
Growing trees.. Progress: 21%. Estimated remaining time: 2 minutes, 0 seconds.
Growing trees.. Progress: 43%. Estimated remaining time: 1 minute, 23 seconds.
Growing trees.. Progress: 65%. Estimated remaining time: 52 seconds.
Growing trees.. Progress: 86%. Estimated remaining time: 20 seconds.
# print the model summary
print("Random Forest Model Summary:")
[1] "Random Forest Model Summary:"
print(imp.rf)
Ranger result

Call:
 ranger(formula = int_rate ~ ., data = imp.train_data, num.trees = 500,      verbose = TRUE, importance = "impurity", oob.error = TRUE) 

Type:                             Regression 
Number of trees:                  500 
Sample size:                      633865 
Number of independent variables:  20 
Mtry:                             4 
Target node size:                 5 
Variable importance mode:         impurity 
Splitrule:                        variance 
OOB prediction error (MSE):       8.372169 
R squared (OOB):                  0.538151 
# make predictions on the training and testing data
imp.rf.train_predictions <- predict(imp.rf, data = imp.train_data)
imp.rf.test_predictions <- predict(imp.rf, data = imp.test_data)

# calculate Mean Squared Error (MSE) for training and testing
imp.rf.train_mse <- mean((imp.rf.train_predictions$predictions - imp.train_data$int_rate)^2)
imp.rf.test_mse <- mean((imp.rf.test_predictions$predictions - imp.test_data$int_rate)^2)

# calculate Root Mean Squared Error (RMSE) for training and testing
imp.rf.train_rmse <- sqrt(imp.rf.train_mse)
imp.rf.test_rmse <- sqrt(imp.rf.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
imp.rf.train_mae <- mean(abs(imp.rf.train_predictions$predictions - imp.train_data$int_rate))
imp.rf.test_mae <- mean(abs(imp.rf.test_predictions$predictions - imp.test_data$int_rate))

# calculate R-squared (R²) for training and testing
imp.rf.train_r2 <- 1 - (sum((imp.train_data$int_rate - imp.rf.train_predictions$predictions)^2) / sum((imp.train_data$int_rate - mean(imp.train_data$int_rate))^2))
imp.rf.test_r2 <- 1 - (sum((test_data$int_rate - rf.test_predictions$predictions)^2) / sum((imp.test_data$int_rate - mean(imp.test_data$int_rate))^2))

# display the metrics
cat("Training MSE:", imp.rf.train_mse, "\n")
Training MSE: 1.63843 
cat("Testing MSE:", imp.rf.test_mse, "\n")
Testing MSE: 8.36116 
cat("Training RMSE:", imp.rf.train_rmse, "\n")
Training RMSE: 1.280012 
cat("Testing RMSE:", imp.rf.test_rmse, "\n")
Testing RMSE: 2.891567 
cat("Training MAE:", imp.rf.train_mae, "\n")
Training MAE: 1.005422 
cat("Testing MAE:", imp.rf.test_mae, "\n")
Testing MAE: 2.296878 
cat("Training R-squared (R²):", imp.rf.train_r2, "\n")
Training R-squared (R²): 0.9096162 
cat("Testing R-squared (R²):", imp.rf.test_r2, "\n")
Testing R-squared (R²): 0.5410509 
#### Boosting with only importance variables ####

# define the target variable for training and testing
imp.xgb.y_train <- imp.train_data$int_rate
imp.xgb.y_test <- imp.test_data$int_rate

# define the feature matrix for training and testing (exclude the target variable)
imp.xgb.X_train <- imp.train_data[, -which(names(imp.train_data) == 'int_rate')]
imp.xgb.X_test <- imp.test_data[, -which(names(imp.test_data) == 'int_rate')]

# fit a gradient boosting regression model using xgboost
imp.xgb <- xgboost(
  data = as.matrix(imp.xgb.X_train),
  label = imp.xgb.y_train,
  nrounds = 100,
  verbose = 0
)

# make predictions on the training and testing data
imp.xgb.train_predictions <- predict(imp.xgb, newdata = as.matrix(imp.xgb.X_train))
imp.xgb.test_predictions <- predict(imp.xgb, newdata = as.matrix(imp.xgb.X_test))

# calculate Mean Squared Error (MSE) for training and testing
imp.xgb.train_mse <- mean((imp.xgb.train_predictions - imp.xgb.y_train)^2)
imp.xgb.test_mse <- mean((imp.xgb.test_predictions - imp.xgb.y_test)^2)

# calculate Root Mean Squared Error (RMSE) for training and testing
imp.xgb.train_rmse <- sqrt(imp.xgb.train_mse)
imp.xgb.test_rmse <- sqrt(imp.xgb.test_mse)

# calculate Mean Absolute Error (MAE) for training and testing
imp.xgb.train_mae <- mean(abs(imp.xgb.train_predictions - imp.xgb.y_train))
imp.xgb.test_mae <- mean(abs(imp.xgb.test_predictions - imp.xgb.y_test))

# calculate R-squared (R²) for training and testing
imp.xgb.train_r2 <- 1 - (sum((imp.xgb.y_train - imp.xgb.train_predictions)^2) / sum((imp.xgb.y_train - mean(imp.xgb.y_train))^2))
imp.xgb.test_r2 <- 1 - (sum((imp.xgb.y_test - imp.xgb.test_predictions)^2) / sum((imp.xgb.y_test - mean(imp.xgb.y_test))^2))

# display the metrics
cat("Training MSE:", imp.xgb.train_mse, "\n")
Training MSE: 7.463161 
cat("Testing MSE:", imp.xgb.test_mse, "\n")
Testing MSE: 7.790172 
cat("Training RMSE:", imp.xgb.train_rmse, "\n")
Training RMSE: 2.731879 
cat("Testing RMSE:", imp.xgb.test_rmse, "\n")
Testing RMSE: 2.791088 
cat("Training MAE:", imp.xgb.train_mae, "\n")
Training MAE: 2.154894 
cat("Testing MAE:", imp.xgb.test_mae, "\n")
Testing MAE: 2.198685 
cat("Training R-squared (R²):", imp.xgb.train_r2, "\n")
Training R-squared (R²): 0.5882955 
cat("Testing R-squared (R²):", imp.xgb.test_r2, "\n")
Testing R-squared (R²): 0.5716244 

The dataset was filtered by the 20 variables with the most importance (from the rf results). As we can see above, the errors of each model are more or less the errors with the double variables we had before, so filtering by these 20 “important variables” does not seem making sense…

Hyperparameter Tuning for XGBoosting:

# define the number of cores
numCores <- detectCores() - 1

# register doParallel as the backend for parallel execution
registerDoParallel(cores=numCores)

# define the control using a cross-validation approach
train_control <- trainControl(method = "cv", number = 5, verboseIter = TRUE)

# define the grid of hyperparameters to search over
xgb.grid <- expand.grid(
  nrounds = c(100, 200, 300),
  eta = c(0.01, 0.05, 0.1),
  max_depth = c(3, 6, 9),
  gamma = c(0, 0.1, 0.2),
  colsample_bytree = c(0.5, 0.8, 1),
  min_child_weight = c(1, 5, 10),
  subsample = c(0.5, 0.75, 1)
)

# train the model
xgb.tuned <- train(
  x = train_data, y = xgb.y_train,
  method = "xgbTree",
  trControl = train_control,
  tuneGrid = xgb.grid
)

# view the best tuning parameters
print(xgb.tuned$bestTune)

# stop the parallel backend
stopImplicitCluster()

Saving our best model:

saveRDS(xgb, file = "best_model_xgb.rds")
LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OgogIGh0bWxfbm90ZWJvb2s6IGRlZmF1bHQKICBwZGZfZG9jdW1lbnQ6IGRlZmF1bHQKLS0tCgojIERhdGEgUHJlLXByb2Nlc3NpbmcKCkxvYWQgbmVlZGVkIGxpYnJhcmllcwoKYGBge3J9CmxpYnJhcnkocmVhZHIpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShkcGx5cikKbGlicmFyeShjYXJldCkKbGlicmFyeShnbG1uZXQpCmxpYnJhcnkoYm9vdCkKbGlicmFyeSh0cmVlKQpsaWJyYXJ5KHJhbmdlcikKbGlicmFyeSh4Z2Jvb3N0KQpsaWJyYXJ5KGdibSkKbGlicmFyeSh2aXApCmxpYnJhcnkoSVNMUikKbGlicmFyeSh0aWR5cikKbGlicmFyeShncmlkRXh0cmEpCmxpYnJhcnkocmVzaGFwZTIpCmBgYApTZXQgdGhlIHNlZWQgZm9yIHJlcHJvZHVjaWJpbGl0eQoKYGBge3J9CnNldC5zZWVkKDEpCmBgYAoKTG9hZCB0aGUgZGF0YXNldAoKYGBge3J9Cm9yaWdpbmFsX2xjX2RhdGEgPC0gcmVhZC5jc3YoIkxDZGF0YS5jc3YiLHNlcCA9ICI7IikKbGNfZGF0YSA8LSBvcmlnaW5hbF9sY19kYXRhCmBgYAoKUmVtb3ZlIGF0dHJpYnV0ZXMgbm90IGF2YWlsYWJsZSBmb3IgcHJlZGljdGlvbgoKYGBge3J9CmxjX2RhdGEgPC0gc3Vic2V0KGxjX2RhdGEsIHNlbGVjdCA9IC1jKGNvbGxlY3Rpb25fcmVjb3ZlcnlfZmVlLCBpbnN0YWxsbWVudCwgaXNzdWVfZCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFzdF9weW1udF9hbW50LCBsYXN0X3B5bW50X2QsIGxvYW5fc3RhdHVzLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBuZXh0X3B5bW50X2QsIG91dF9wcm5jcCwgb3V0X3BybmNwX2ludiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHltbnRfcGxhbiwgcmVjb3ZlcmllcywgdG90YWxfcHltbnQsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvdGFsX3B5bW50X2ludix0b3RhbF9yZWNfaW50LCB0b3RhbF9yZWNfbGF0ZV9mZWUsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0b3RhbF9yZWNfcHJuY3ApKQpgYGAKCmBgYHtyfQpzdW1tYXJ5KGxjX2RhdGEpCmBgYApGaXJzdCB3ZSBkZWxldGUgdGhlIGNvbHVtbnMgd2hpY2ggYXJlbid0IHVzZWZ1bCBmb3Igb3VyIHByZWRpY3Rpb24KCmBgYHtyfQpsY19kYXRhJGlkIDwtIE5VTEwKbGNfZGF0YSRtZW1iZXJfaWQgPC0gTlVMTApsY19kYXRhJHppcF9jb2RlIDwtIE5VTEwKbGNfZGF0YSR1cmwgPC0gTlVMTApgYGAKCkxvb2tzIGxpa2UgKipwb2xpY3lfY29kZSoqIGNvbnRhaW5zIGp1c3QgdmFsdWUgZXF1YWwgdG8gMSwgaXQgY2FuIGJlIHJlbW92ZWQKCmBgYHtyfQpsY19kYXRhJHBvbGljeV9jb2RlIDwtIE5VTEwKYGBgCgpSZW1vdmUgYWRkaXRpb25hbCBjb2x1bW5zIHdoaWNoIGFyZSByZWxhdGVkIHRvIHRoZSBoaXN0b3JpY2FsIGRhdGEKCmBgYHtyfQpsY19kYXRhJGxhc3RfY3JlZGl0X3B1bGxfZCA8LSBOVUxMCmBgYAoKVGhlbiB3ZSBkZWxldGUgdGhlIGNvbHVtbnMgd2hpY2ggY2FuJ3QgYmUgY29udmVydGVkIHRvIGNhdGVnb3JpY2FsIGFuZCByZXF1aXJlIE5MUAoKYGBge3J9CmxjX2RhdGEkdGl0bGUgPC0gTlVMTApsY19kYXRhJGRlc2MgPC0gTlVMTApsY19kYXRhJGVtcF90aXRsZSA8LSBOVUxMCmBgYAoKTGV0J3MgZXhhbWluZSB0aGUgKipsb2FuX2FtbnQqKiBjb2x1bW4KCmBgYHtyfQpzdW0oaXMubmEobGNfZGF0YSRsb2FuX2FtbnQpKQpjb3IobGNfZGF0YSRsb2FuX2FtbnQsIGxjX2RhdGEkaW50X3JhdGUpCmhpc3QobGNfZGF0YSRsb2FuX2FtbnQsIGJyZWFrcyA9IDIwLCBtYWluID0gImxvYW5fYW1udCBkaXN0cmlidXRpb24iLCB4bGFiID0gImxvYW5fYW1udCIsIGNvbCA9ICJsaWdodGJsdWUiLCBib3JkZXIgPSAiYmxhY2siKQpnZ3Bsb3QoZGF0YSA9IGxjX2RhdGEsIG1hcHBpbmcgPSBhZXMoeD1pbnRfcmF0ZSx5PWxvYW5fYW1udCkpICsgZ2VvbV9ib3hwbG90KCkKYGBgCgpTdGFuZGFyZGl6ZSAqKmxvYW5fYW1udCoqCgpgYGB7cn0KI2xjX2RhdGEkbG9hbl9hbW50IDwtIHNjYWxlKGxjX2RhdGEkbG9hbl9hbW50KQpgYGAKCkxldCdzIGV4YW1pbmUgdGhlICoqZnVuZGVkX2FtbnQqKiBjb2x1bW4KCmBgYHtyfQpzdW0oaXMubmEobGNfZGF0YSRmdW5kZWRfYW1udCkpCmNvcihsY19kYXRhJGZ1bmRlZF9hbW50LCBsY19kYXRhJGludF9yYXRlKQpoaXN0KGxjX2RhdGEkZnVuZGVkX2FtbnQsIGJyZWFrcyA9IDIwLCBtYWluID0gImZ1bmRlZF9hbW50IGRpc3RyaWJ1dGlvbiIsIHhsYWIgPSAiZnVuZGVkX2FtbnQiLCBjb2wgPSAibGlnaHRibHVlIiwgYm9yZGVyID0gImJsYWNrIikKYGBgCgpBcyB3ZSBjYW4gc2VlLCAqKmZ1bmRlZF9hbW50KiogaXMgYWxtb3N0IHRoZSBzYW1lIGFzIHRoZSAqKmxvYW5fYW1udCoqIGNvbHVtbiwgY29uc2VxdWVudGx5LCB3ZSByZW1vdmUgaXQuCgpgYGB7cn0KbGNfZGF0YSRmdW5kZWRfYW1udCA8LSBOVUxMIApgYGAKCkxldCdzIGV4YW1pbmUgdGhlICoqZnVuZGVkX2FtbnRfaW52KiogY29sdW1uCgpgYGB7cn0Kc3VtKGlzLm5hKGxjX2RhdGEkZnVuZGVkX2FtbnRfaW52KSkKY29yKGxjX2RhdGEkZnVuZGVkX2FtbnRfaW52LCBsY19kYXRhJGludF9yYXRlKQpoaXN0KGxjX2RhdGEkZnVuZGVkX2FtbnRfaW52LCBicmVha3MgPSAyMCwgbWFpbiA9ICJmdW5kZWRfYW1udF9pbnYgZGlzdHJpYnV0aW9uIiwgeGxhYiA9ICJmdW5kZWRfYW1udF9pbnYiLCBjb2wgPSAibGlnaHRibHVlIiwgYm9yZGVyID0gImJsYWNrIikKYGBgCgpSZW1vdmUgKipmdW5kZWRfYW1udF9pbnYqKiBmb3IgdGhlIHNhbWUgcmVhc29uIGFzIGFib3ZlCgpgYGB7cn0KbGNfZGF0YSRmdW5kZWRfYW1udF9pbnYgPC0gTlVMTApgYGAKCkxldCdzIHNlZSB0aGUgKippbnRfcmF0ZSoqIGRpc3RyaWJ1dGlvbi4KYGBge3J9Cmhpc3QobGNfZGF0YSRpbnRfcmF0ZSwgYnJlYWtzID0gMjAsIG1haW4gPSAiaW50X3JhdGUgZGlzdHJpYnV0aW9uIiwgeGxhYiA9ICJpbnRfcmF0ZSIsIGNvbCA9ICJsaWdodGJsdWUiLCBib3JkZXIgPSAiYmxhY2siKQpgYGAKClN0YW5kYXJkaXplIGludCByYXRlOgpgYGB7cn0KI2xjX2RhdGEkaW50X3JhdGUgPC0gc2NhbGUobGNfZGF0YSRpbnRfcmF0ZSkKYGBgCgpBcyB3ZSBjYW4gb2JzZXJ2ZSwgdGhlcmUgYXJlIDQwMzYzIE5Bcy4gV2UgY2FuIGFzc3VtZSA0MDM2MyBkbyBub3Qgd29yay4KYGBge3J9CmJhcnBsb3QodGFibGUobGNfZGF0YSRlbXBfbGVuZ3RoKSwKICAgICAgICB4bGFiID0gImVtcF9sZW5ndGggeWVhcnMiLCAKICAgICAgICB5bGFiID0gIkZyZXF1ZW5jeSIsIAogICAgICAgIGNvbCA9ICJza3libHVlIiwgCiAgICAgICAgYm9yZGVyID0gImJsYWNrIiwKICAgICAgICBjZXgubmFtZXMgPSAwLjYpICAjIFRoZSBzaXplIG9mIHRoZSBtYWluIHRpdGxlCmBgYAoKU2luY2UgKiplbXBfbGVuZ3RoKiogc2VlbXMgdG8gYmUgY2F0ZWdvcmljYWwsIHdlIHRyYW5zZm9ybSBpdCB0byBhcyBhIGZhY3RvciBhbmQgdGhlbiBhcyBudW1lcmljLgpUaGUgY29udmVyc2lvbiB0byBudW1lcmljIGlzIG5lZWRlZCBmb3Igc3VwcG9ydGluZyB0aGUgRGVjaXNpb24gVHJlZXMgYW5kIFhHQm9vc3QKCmBgYHtyfQpsY19kYXRhJGVtcF9sZW5ndGggPC0gYXMuZmFjdG9yKGxjX2RhdGEkZW1wX2xlbmd0aCkKZ2dwbG90KGRhdGEgPSBsY19kYXRhLCBtYXBwaW5nID0gYWVzKHg9aW50X3JhdGUseT1lbXBfbGVuZ3RoKSkgKyBnZW9tX2JveHBsb3QoKQpsY19kYXRhJGVtcF9sZW5ndGggPC0gYXMubnVtZXJpYyhsY19kYXRhJGVtcF9sZW5ndGgpCmBgYAoKQXMgd2UgY2FuIHNlZSwgKip0ZXJtKiogcGxheXMgYSBjcnVjaWFsIHJvbGUgaW4gcHJlZGljdGluZyB0aGUgaW50ZXJlc3QgcmF0ZS4KCmBgYHtyfQpsY19kYXRhJHRlcm0gPC0gYXMuZmFjdG9yKGxjX2RhdGEkdGVybSkKZ2dwbG90KGRhdGEgPSBsY19kYXRhLCBtYXBwaW5nID0gYWVzKHg9aW50X3JhdGUseT10ZXJtKSkgKyBnZW9tX2JveHBsb3QoKQpsY19kYXRhJHRlcm0gPC0gYXMubnVtZXJpYyhsY19kYXRhJHRlcm0pCmBgYAoKQ2xlYW5pbmcgb2YgKipob21lX293bmVyc2hpcCoqOgoKRHVyaW5nIHRoZSBkYXRhIGNsZWFuaW5nIHBoYXNlLCBvdXIgYW5hbHlzaXMgcmV2ZWFsZWQgdGhhdCB0aGUgdmFyaWFibGUgImhvbWVfb3duZXJzaGlwIiBkb2VzIG5vdCBzaG93IGEgZGlzdGluY3QgY29ycmVsYXRpb24gd2l0aCBpbnRlcmVzdCByYXRlcy4gU3BlY2lmaWNhbGx5LCBhbW9uZyB0aGUgY2F0ZWdvcmllcywgIkFOWSIgYW5kICJPVEhFUiIgY29udGFpbiAyIGFuZCAxNTQgY2FzZXMsIHJlc3BlY3RpdmVseSwgd2hpbGUgdGhlICJOT05FIiBjYXRlZ29yeSBjb21wcmlzZXMgMzkgY2FzZXMuIEFsdGhvdWdoIHRoZSAiTk9ORSIgY2F0ZWdvcnkgYXBwZWFycyB0byBkZW1vbnN0cmF0ZSBhIGhpZ2hlciBpbnRlcmVzdCByYXRlIGNvbXBhcmVkIHRvIG90aGVycywgdGhlIGxpbWl0ZWQgc2FtcGxlIHNpemUgb2YgMzkgY2FzZXMgcmFpc2VzIGRvdWJ0cyBhYm91dCB0aGUgcmVsaWFiaWxpdHkgb2YgdGhpcyBvYnNlcnZhdGlvbi4gTm90YWJseSwgdGhlICJOT05FIiBjYXRlZ29yeSBtaWdodCBwZXJ0YWluIHRvIGluZGl2aWR1YWxzIGV4cGVyaWVuY2luZyBob21lbGVzc25lc3MsIHByb21wdGluZyBldGhpY2FsIGNvbmNlcm5zIGFib3V0IGxvYW4gcHJvdmlzaW9uIHRvIHRoaXMgZGVtb2dyYXBoaWMuCgpgYGB7cn0KdGFibGUobGNfZGF0YSRob21lX293bmVyc2hpcCkKZ2dwbG90KGRhdGEgPSBsY19kYXRhLCBtYXBwaW5nID0gYWVzKHg9aW50X3JhdGUseT1ob21lX293bmVyc2hpcCkpICsgZ2VvbV9ib3hwbG90KCkKYGBgCgpUaGVuLCB3ZSByZXRhaW4gbW9ydGdhZ2UsIG93biBhbmQgcmVudDoKCmBgYHtyfQpsY19kYXRhIDwtIGxjX2RhdGEgJT4lIGZpbHRlcihob21lX293bmVyc2hpcCAlaW4lIGMoIk1PUlRHQUdFIiwiT1dOIiwiUkVOVCIpKQpsY19kYXRhJGhvbWVfb3duZXJzaGlwIDwtIGFzLm51bWVyaWMoYXMuZmFjdG9yKGxjX2RhdGEkaG9tZV9vd25lcnNoaXApKQpgYGAKCk1vc3Qgb2YgdGhlIGxvYW4gYXBwbGljYXRpb25zIGFyZSBJbmRpdmlkdWFsLCB0aGlzIG1lYW5zIHRoYXQgbW9zdCBvZiB0aGUgdmFsdWVzIG9mIHRoZSBjb2x1bW5zICoqZHRpX2pvaW50KiosKiphbm51YWxfaW5jX2pvaW50KiogYW5kICoqdmVyaWZpY2F0aW9uX3N0YXR1c19qb2ludCoqIGFyZSBOdWxsLgpXZSB3b3VsZCBsaWtlIHRvIGtlZXAgdGhlIGluZm9ybWF0aW9uIGFib3V0IEpvaW50IGxvYW5zLCB0aGlzIG1lYW5zIHRoYXQgd2UgY2FuIHJlcGxhY2UgdGhlIE51bGwgdmFsdWVzIHdpdGggMC4KCmBgYHtyfQpuYXYgPC0gYygnJywgJyAnKQpsY19kYXRhIDwtIHRyYW5zZm9ybShsY19kYXRhLCB2ZXJpZmljYXRpb25fc3RhdHVzX2pvaW50PXJlcGxhY2UodmVyaWZpY2F0aW9uX3N0YXR1c19qb2ludCwgdmVyaWZpY2F0aW9uX3N0YXR1c19qb2ludCAlaW4lIG5hdiwgTkEpKQpsY19kYXRhIDwtCiAgbGNfZGF0YSAlPiUKICBtdXRhdGUoZHRpX2pvaW50ID0gaWZlbHNlKGlzLm5hKGR0aV9qb2ludCkgPT0gVFJVRSwgMCwgZHRpX2pvaW50KSkgJT4lCiAgbXV0YXRlKGFubnVhbF9pbmNfam9pbnQgPSBpZmVsc2UoaXMubmEoYW5udWFsX2luY19qb2ludCkgPT0gVFJVRSwgMCwgYW5udWFsX2luY19qb2ludCkpICU+JQogIG11dGF0ZSh2ZXJpZmljYXRpb25fc3RhdHVzX2pvaW50ID0gaWZlbHNlKGlzLm5hKHZlcmlmaWNhdGlvbl9zdGF0dXNfam9pbnQpID09IFRSVUUsICdOQScsIHZlcmlmaWNhdGlvbl9zdGF0dXNfam9pbnQpKQoKYGBgCgpUaGUgZW1wdHkgc3RyaW5nIG9yIG51bGwgdmFsdWUgaW4gKip2ZXJpZmljYXRpb25fc3RhdHVzX2pvaW50KiogaXMgcmVwbGFjZWQgc3VjY2Vzc2Z1bGx5LgoKYGBge3J9CnRhYmxlKGxjX2RhdGEkdmVyaWZpY2F0aW9uX3N0YXR1cykKdGFibGUobGNfZGF0YSR2ZXJpZmljYXRpb25fc3RhdHVzX2pvaW50KQpgYGAKVGhlbiAqKnZlcmlmaWNhdGlvbl9zdGF0dXNfam9pbnQqKiBhbmQgKip2ZXJpZmljYXRpb25fc3RhdHVzKiogY29sdW1ucyBhcmUgY29udmVydGVkIGluIGNhdGVnb3JpY2FsIGFuZCB0aGVuIG51bWVyaWNhbCB2YWx1ZS4KVGhlIGNvbHVtbiAqKmFwcGxpY2F0aW9uX3R5cGUqKiBpcyBvYnNvbGV0ZSwgc2luY2UgdGhlIGluZm9ybWF0aW9uIGFib3V0IHdoZXRoZXIgdGhlIGxvYW4gaXMgaW5kaXZpZHVhbCBvciBqb2ludCBpcyBhbHJlYWR5IGNvbnRhaW5lZCBpbiB0aGUgcHJldmlvdXMgdmFyaWFibGVzLgpgYGB7cn0KbGNfZGF0YSR2ZXJpZmljYXRpb25fc3RhdHVzIDwtIGFzLm51bWVyaWMoYXMuZmFjdG9yKGxjX2RhdGEkdmVyaWZpY2F0aW9uX3N0YXR1cykpCmxjX2RhdGEkdmVyaWZpY2F0aW9uX3N0YXR1c19qb2ludCA8LSBhcy5udW1lcmljKGFzLmZhY3RvcihsY19kYXRhJHZlcmlmaWNhdGlvbl9zdGF0dXNfam9pbnQpKQpsY19kYXRhIDwtIGxjX2RhdGEgJT4lIHNlbGVjdCgtYXBwbGljYXRpb25fdHlwZSkKYGBgCgoKTGV0J3MgY2hlY2sgaWYgb3RoZXIgaXMgTkEgb3IgYSByZWFsIHZhbHVlIGZvciBwdXJwb3NlLiBJdCdzIGEgcmVhbCBvbmUsIHNvIHdlIGRvbid0IGhhdmUgdG8gaGFuZGxlIGl0LgpgYGB7cn0KbGNfZGF0YSRwdXJwb3NlIDwtIGFzLmZhY3RvcihsY19kYXRhJHB1cnBvc2UpCmdncGxvdChkYXRhID0gbGNfZGF0YSwgbWFwcGluZyA9IGFlcyh4PWludF9yYXRlLHk9cHVycG9zZSkpICsgZ2VvbV9ib3hwbG90KCkKbGNfZGF0YSRwdXJwb3NlIDwtIGFzLm51bWVyaWMobGNfZGF0YSRwdXJwb3NlKQpgYGAKCkxldCdzIGhhdmUgYSBnbGFuY2UgdG8gdGhlIHN0YXRlIGFkZHJlc3M6CmBgYHtyfQp0YWJsZShsY19kYXRhJGFkZHJfc3RhdGUpCmxjX2RhdGEkYWRkcl9zdGF0ZSA8LSBhcy5mYWN0b3IobGNfZGF0YSRhZGRyX3N0YXRlKQpnZ3Bsb3QoZGF0YSA9IGxjX2RhdGEsIG1hcHBpbmcgPSBhZXMoeD1pbnRfcmF0ZSx5PWFkZHJfc3RhdGUpKSArIGdlb21fYm94cGxvdCgpCmxjX2RhdGEkYWRkcl9zdGF0ZSA8LSBhcy5udW1lcmljKGxjX2RhdGEkYWRkcl9zdGF0ZSkKYGBgCgpSZWdhcmRpbmcgZGVsaW5xdWVuY3kgaW4gdGhlIGxhc3QgMiB5ZWFycywgdGhlcmUgYXJlIGZldyBOQXMgdGhlbiByZW1vdmUgdGhlbToKYGBge3J9CmxjX2RhdGEgPC0gbGNfZGF0YSAlPiUgCiAgICBmaWx0ZXIoIShpcy5uYShkZWxpbnFfMnlycykpKQpgYGAKClRoZSBjb2x1bW5zICoqbXRoc19zaW5jZV9kZWxpbnFfY2F0KiosICoqbXRoc19zaW5jZV9sYXN0X3JlY29yZCoqLCAqKm10aHNfc2luY2VfcmNudF9pbCoqIGFuZCAqKm10aHNfc2luY2VfbGFzdF9tYWpvcl9kZXJvZyoqIGNvbnRhaW4gbnVtZXJpY2FsIHZhbHVlcyB3aGljaCByZWZlciB0byB0aGUgbnVtYmVyIG9mIHRoZSBtb250aHMuIFNpbmNlIHRoaXMgY29sdW1ucyBjb250YWluIGEgbG90IG9mIG51bGwgdmFsdWVzIHdoaWNoIGNhbid0IGJlIHJlcGxhY2VkIHdpdGggMCdzLCBvbmUgb2YgdGhlIG1vc3QgYXBwcm9wcmlhdGUgb3BlcmF0aW9ucyB0aGF0IGNhbiBiZSBtYWRlIGlzIGFwcGx5aW5nIGRpc2NyZXRpemF0aW9uLiBXZSBkbyB0aGlzIGJ5IGNyZWF0aW5nIGEgc2V0IG9mIGNvbnRpZ3VvdXMgYmlucyBiYXNlZCBvbiB5ZWFycywgd2hpbGUgZm9yIHRoZSBudWxsIHZhbHVlcyB3ZSBjcmVhdGUgYSBzZXBhcmF0ZSBiaW4uCgpgYGB7cn0KbGNfZGF0YSA8LSBsY19kYXRhICU+JQogIG11dGF0ZShtdGhzX3NpbmNlX2RlbGlucV9jYXQgPSBpZmVsc2UoCiAgICBpcy5uYShtdGhzX3NpbmNlX2xhc3RfZGVsaW5xKSA9PSBUUlVFLAogICAgIk5PTkUiLAogICAgaWZlbHNlKAogICAgICBtdGhzX3NpbmNlX2xhc3RfZGVsaW5xIDw9IDEyLAogICAgICAiTGVzc18xX1kiLAogICAgICBpZmVsc2UoCiAgICAgICAgbXRoc19zaW5jZV9sYXN0X2RlbGlucSA8PSAyNCwKICAgICAgICAiTGVzc18yX1kiLAogICAgICAgIGlmZWxzZSgKICAgICAgICAgIG10aHNfc2luY2VfbGFzdF9kZWxpbnEgPD0gMzYsCiAgICAgICAgICAiTGVzc18zX1kiLAogICAgICAgICAgaWZlbHNlKG10aHNfc2luY2VfbGFzdF9kZWxpbnEgPD0gNDgsICJMZXNzXzRfWSIsICJNb3JlXzRfWSIpCiAgICAgICAgKQogICAgICApCiAgICApCiAgKSkgJT4lIHNlbGVjdCgtbXRoc19zaW5jZV9sYXN0X2RlbGlucSkKICAgICAgICAgIApsY19kYXRhJG10aHNfc2luY2VfZGVsaW5xX2NhdCA8LSBhcy5mYWN0b3IobGNfZGF0YSRtdGhzX3NpbmNlX2RlbGlucV9jYXQpCmdncGxvdChkYXRhID0gbGNfZGF0YSwgbWFwcGluZyA9IGFlcyh4PWludF9yYXRlLHk9bXRoc19zaW5jZV9kZWxpbnFfY2F0KSkrZ2VvbV9ib3hwbG90KCkKbGNfZGF0YSRtdGhzX3NpbmNlX2RlbGlucV9jYXQgPC0gYXMubnVtZXJpYyhsY19kYXRhJG10aHNfc2luY2VfZGVsaW5xX2NhdCkKYGBgCgpgYGB7cn0KbGNfZGF0YSA8LSBsY19kYXRhICU+JQogIG11dGF0ZShtdGhzX3NpbmNlX2xhc3RfcmVjb3JkX2NhdCA9IGlmZWxzZSgKICAgIGlzLm5hKG10aHNfc2luY2VfbGFzdF9yZWNvcmQpID09IFRSVUUsCiAgICAiTk9ORSIsCiAgICBpZmVsc2UoCiAgICAgIG10aHNfc2luY2VfbGFzdF9yZWNvcmQgPD0gMTIsCiAgICAgICJMZXNzXzFfWSIsCiAgICAgIGlmZWxzZSgKICAgICAgICBtdGhzX3NpbmNlX2xhc3RfcmVjb3JkIDw9IDI0LAogICAgICAgICJMZXNzXzJfWSIsCiAgICAgICAgaWZlbHNlKAogICAgICAgICAgbXRoc19zaW5jZV9sYXN0X3JlY29yZCA8PSAzNiwKICAgICAgICAgICJMZXNzXzNfWSIsCiAgICAgICAgICBpZmVsc2UobXRoc19zaW5jZV9sYXN0X3JlY29yZCA8PSA0OCwgIkxlc3NfNF9ZIiwgIk1vcmVfNF9ZIikKICAgICAgICApCiAgICAgICkKICAgICkKICApKSAlPiUgc2VsZWN0KC1tdGhzX3NpbmNlX2xhc3RfcmVjb3JkKQoKbGNfZGF0YSRtdGhzX3NpbmNlX2xhc3RfcmVjb3JkX2NhdCA8LSBhcy5mYWN0b3IobGNfZGF0YSRtdGhzX3NpbmNlX2xhc3RfcmVjb3JkX2NhdCkKZ2dwbG90KGRhdGEgPSBsY19kYXRhLCBtYXBwaW5nID0gYWVzKHg9aW50X3JhdGUseT1tdGhzX3NpbmNlX2xhc3RfcmVjb3JkX2NhdCkpK2dlb21fYm94cGxvdCgpCmxjX2RhdGEkbXRoc19zaW5jZV9sYXN0X3JlY29yZF9jYXQgPC0gYXMubnVtZXJpYyhsY19kYXRhJG10aHNfc2luY2VfbGFzdF9yZWNvcmRfY2F0KQpgYGAKCmBgYHtyfQpsY19kYXRhIDwtbGNfZGF0YSAlPiUgCiAgbXV0YXRlKG10aHNfc2luY2VfcmNudF9pbF9jYXQgPSAgaWZlbHNlKAogICAgaXMubmEobXRoc19zaW5jZV9yY250X2lsKSA9PSBUUlVFLAogICAgIk5PTkUiLAogICAgaWZlbHNlKAogICAgICBtdGhzX3NpbmNlX3JjbnRfaWwgPD0gMTIsCiAgICAgICJMZXNzXzFfWSIsCiAgICAgIGlmZWxzZSgKICAgICAgICBtdGhzX3NpbmNlX3JjbnRfaWwgPD0gMjQsCiAgICAgICAgIkxlc3NfMl9ZIiwKICAgICAgICBpZmVsc2UoCiAgICAgICAgICBtdGhzX3NpbmNlX3JjbnRfaWwgPD0gMzYsCiAgICAgICAgICAiTGVzc18zX1kiLAogICAgICAgICAgaWZlbHNlKG10aHNfc2luY2VfcmNudF9pbCA8PSA0OCwgIkxlc3NfNF9ZIiwgIk1vcmVfNF9ZIikKICAgICAgICApCiAgICAgICkKICAgICkKICApKSAlPiUgc2VsZWN0KC1tdGhzX3NpbmNlX3JjbnRfaWwpCgpsY19kYXRhJG10aHNfc2luY2VfcmNudF9pbF9jYXQgPC0gYXMuZmFjdG9yKGxjX2RhdGEkbXRoc19zaW5jZV9yY250X2lsX2NhdCkKZ2dwbG90KGRhdGEgPSBsY19kYXRhLCBtYXBwaW5nID0gYWVzKHg9aW50X3JhdGUseT1tdGhzX3NpbmNlX3JjbnRfaWxfY2F0KSkrZ2VvbV9ib3hwbG90KCkKbGNfZGF0YSRtdGhzX3NpbmNlX3JjbnRfaWxfY2F0IDwtIGFzLm51bWVyaWMobGNfZGF0YSRtdGhzX3NpbmNlX3JjbnRfaWxfY2F0KQpgYGAKCmBgYHtyfQpsY19kYXRhIDwtbGNfZGF0YSAlPiUgCiAgbXV0YXRlKG10aHNfc2luY2VfbGFzdF9tYWpvcl9kZXJvZ19jYXQgPSAgaWZlbHNlKAogICAgaXMubmEobXRoc19zaW5jZV9sYXN0X21ham9yX2Rlcm9nKSA9PSBUUlVFLAogICAgIk5PTkUiLAogICAgaWZlbHNlKAogICAgICBtdGhzX3NpbmNlX2xhc3RfbWFqb3JfZGVyb2cgPD0gMTIsCiAgICAgICJMZXNzXzFfWSIsCiAgICAgIGlmZWxzZSgKICAgICAgICBtdGhzX3NpbmNlX2xhc3RfbWFqb3JfZGVyb2cgPD0gMjQsCiAgICAgICAgIkxlc3NfMl9ZIiwKICAgICAgICBpZmVsc2UoCiAgICAgICAgICBtdGhzX3NpbmNlX2xhc3RfbWFqb3JfZGVyb2cgPD0gMzYsCiAgICAgICAgICAiTGVzc18zX1kiLAogICAgICAgICAgaWZlbHNlKG10aHNfc2luY2VfbGFzdF9tYWpvcl9kZXJvZyA8PSA0OCwgIkxlc3NfNF9ZIiwgIk1vcmVfNF9ZIikKICAgICAgICApCiAgICAgICkKICAgICkKICApKSAlPiUgc2VsZWN0KC1tdGhzX3NpbmNlX2xhc3RfbWFqb3JfZGVyb2cpCgpsY19kYXRhJG10aHNfc2luY2VfbGFzdF9tYWpvcl9kZXJvZ19jYXQgPC0gYXMuZmFjdG9yKGxjX2RhdGEkbXRoc19zaW5jZV9sYXN0X21ham9yX2Rlcm9nX2NhdCkKZ2dwbG90KGRhdGEgPSBsY19kYXRhLCBtYXBwaW5nID0gYWVzKHg9aW50X3JhdGUseT1tdGhzX3NpbmNlX2xhc3RfbWFqb3JfZGVyb2dfY2F0KSkrZ2VvbV9ib3hwbG90KCkKbGNfZGF0YSRtdGhzX3NpbmNlX2xhc3RfbWFqb3JfZGVyb2dfY2F0IDwtIGFzLm51bWVyaWMobGNfZGF0YSRtdGhzX3NpbmNlX2xhc3RfbWFqb3JfZGVyb2dfY2F0KQoKYGBgCgoKVGhlIHZhcmlhYmxlICoqaW5pdGlhbF9saXN0X3N0YXR1cyoqIGlkZW50aWZpZXMgd2hldGhlciBhIGxvYW4gd2FzIGluaXRpYWxseSBsaXN0ZWQgaW4gdGhlIHdob2xlIChXKSBvciBmcmFjdGlvbmFsIChGKSBtYXJrZXQuIFRoaXMgdmFyaWFibGUgY291bGQgYmUgdXNlZnVsIHNvIHdlIGNhbiBrZWVwIGl0IGFuZCB0cmFuc2Zvcm0gaXQgdG8gYSBmYWN0b3IgYW5kIHRoZW4gdG8gYSBudW1lcmljIHZhbHVlLCBmb3IgdGhlIHNhbWUgcHVycG9zZSBvZiBjb21wYXRpYmlsaXR5IHdpdGggdGhlIFhHQm9vc3QgZnVuY3Rpb24uCgpgYGB7cn0KbGNfZGF0YSRpbml0aWFsX2xpc3Rfc3RhdHVzIDwtIGFzLmZhY3RvcihsY19kYXRhJGluaXRpYWxfbGlzdF9zdGF0dXMpCmdncGxvdChkYXRhID0gbGNfZGF0YSwgbWFwcGluZyA9IGFlcyh4PWludF9yYXRlLHk9aW5pdGlhbF9saXN0X3N0YXR1cykpK2dlb21fYm94cGxvdCgpCmxjX2RhdGEkaW5pdGlhbF9saXN0X3N0YXR1cyA8LSBhcy5udW1lcmljKGxjX2RhdGEkaW5pdGlhbF9saXN0X3N0YXR1cykKYGBgCgpMZXQncyBjaGVjayB3aGljaCBjb2x1bW5zIHN0aWxsIGhhdmUgbnVsbCB2YWx1ZXMKYGBge3J9CmNvbFN1bXMoaXMubmEobGNfZGF0YSkpCmBgYAoKVGhlIGNvbHVtbnMgKipyZXZvbF9iYWwqKiBhbmQgKipyZXZvbF91dGlsKiogY29udGFpbiBvbmx5IGZldyBOQSB2YWx1ZXMsIHRob3NlIHZhbHVlcyBjYW4ndCBiZSByZXBsYWNlZCB3aXRoIDAsIHRoZW4gd2UgZmlsdGVyIHRoZSB2YWx1ZXMgd2hpY2ggYXJlIG5vdCBOQXMuCgpgYGB7cn0KbGNfZGF0YSA8LSBsY19kYXRhICU+JSAKICAgIGZpbHRlcighKGlzLm5hKHJldm9sX2JhbCkpKSAlPiUgCiAgICAgICAgZmlsdGVyKCEoaXMubmEocmV2b2xfdXRpbCkpKQpgYGAKCgpMZXQncyBjaGVjayB3aGljaCBjb2x1bW5zIHN0aWxsIGhhdmUgbnVsbCB2YWx1ZXM6CgpgYGB7cn0KbmFtZXMod2hpY2goY29sU3Vtcyhpcy5uYShsY19kYXRhKSkgPiAwKSkKYGBgCgpSZXBsYWNlIG51bGwgdmFsdWVzIHdpdGggMCB3aGVyZSBpcyBwb3NzaWJsZQoKYGBge3J9CmxjX2RhdGEgPC0KICBsY19kYXRhICU+JQogIG11dGF0ZShvcGVuX2FjY182bSA9IGlmZWxzZShpcy5uYShvcGVuX2FjY182bSkgPT0gVFJVRSwgMCwgb3Blbl9hY2NfNm0pKSAlPiUKICBtdXRhdGUodG90X2N1cl9iYWwgPSBpZmVsc2UoaXMubmEodG90X2N1cl9iYWwpID09IFRSVUUsIDAsIHRvdF9jdXJfYmFsKSkgJT4lCiAgbXV0YXRlKG9wZW5faWxfNm0gPSBpZmVsc2UoaXMubmEob3Blbl9pbF82bSkgPT0gVFJVRSwgMCwgb3Blbl9pbF82bSkpICU+JQogIG11dGF0ZShvcGVuX2lsXzEybSA9IGlmZWxzZShpcy5uYShvcGVuX2lsXzEybSkgPT0gVFJVRSwgMCwgb3Blbl9pbF8xMm0pKSAlPiUKICBtdXRhdGUob3Blbl9pbF8yNG0gPSBpZmVsc2UoaXMubmEob3Blbl9pbF8yNG0pID09IFRSVUUsIDAsIG9wZW5faWxfMjRtKSkgJT4lCiAgbXV0YXRlKHRvdGFsX2JhbF9pbCA9IGlmZWxzZShpcy5uYSh0b3RhbF9iYWxfaWwpID09IFRSVUUsIDAsIHRvdGFsX2JhbF9pbCkpICU+JQogIG11dGF0ZShpbF91dGlsID0gaWZlbHNlKGlzLm5hKGlsX3V0aWwpID09IFRSVUUsIDAsIGlsX3V0aWwpKSAlPiUKICBtdXRhdGUob3Blbl9ydl8xMm0gPSBpZmVsc2UoaXMubmEob3Blbl9ydl8xMm0pID09IFRSVUUsIDAsIG9wZW5fcnZfMTJtKSkgJT4lCiAgbXV0YXRlKHRvdGFsX3Jldl9oaV9saW0gPSBpZmVsc2UoaXMubmEodG90YWxfcmV2X2hpX2xpbSkgPT0gVFJVRSwgMCwgdG90YWxfcmV2X2hpX2xpbSkpICU+JQogIG11dGF0ZShtYXhfYmFsX2JjID0gaWZlbHNlKGlzLm5hKG1heF9iYWxfYmMpID09IFRSVUUsIDAsIG1heF9iYWxfYmMpKSAlPiUKICBtdXRhdGUoYWxsX3V0aWwgPSBpZmVsc2UoaXMubmEoYWxsX3V0aWwpID09IFRSVUUsIDAsIGFsbF91dGlsKSkgJT4lCiAgbXV0YXRlKGlucV9maSA9IGlmZWxzZShpcy5uYShpbnFfZmkpID09IFRSVUUsIDAsIGlucV9maSkpICU+JQogIG11dGF0ZSh0b3RhbF9jdV90bCA9IGlmZWxzZShpcy5uYSh0b3RhbF9jdV90bCkgPT0gVFJVRSwgMCwgdG90YWxfY3VfdGwpKSAlPiUKICBtdXRhdGUoaW5xX2xhc3RfMTJtID0gaWZlbHNlKGlzLm5hKGlucV9sYXN0XzEybSkgPT0gVFJVRSwgMCwgaW5xX2xhc3RfMTJtKSkgJT4lCiAgbXV0YXRlKG9wZW5fcnZfMjRtID0gaWZlbHNlKGlzLm5hKG9wZW5fcnZfMjRtKSA9PSBUUlVFLCAwLCBvcGVuX3J2XzI0bSkpICU+JQogIG11dGF0ZSh0b3RfY29sbF9hbXQgPSBpZmVsc2UoaXMubmEodG90X2NvbGxfYW10KT09IFRSVUUsMCwgdG90X2NvbGxfYW10KSkgJT4lCiAgbXV0YXRlKGNvbGxlY3Rpb25zXzEyX210aHNfZXhfbWVkID0gaWZlbHNlKGlzLm5hKGNvbGxlY3Rpb25zXzEyX210aHNfZXhfbWVkKT09IFRSVUUsMCwgY29sbGVjdGlvbnNfMTJfbXRoc19leF9tZWQpKQpgYGAKCioqZWFybGllc3RfY3JfbGluZSoqIGNvbnRhaW5zIHRoZSBtb250aCB0aGUgYm9ycm93ZXIncyBlYXJsaWVzdCByZXBvcnRlZCBjcmVkaXQgbGluZSB3YXMgb3BlbmVkLgpFdmVuIGlmIHRoaXMgZGF0ZSBjb25zaXN0cyBvbmx5IG9uIG1vbnRoIGFuZCB5ZWFyLCBzdGlsbCB0aGVyZSBhcmUgdG9vIG1hbnkgdW5pcXVlIHZhbHVlcy4KV2UgY291bGQgdHJhbnNmb3JtIHRoZSBkYXRlcyBpbiB0byBhIG51bWVyaWNhbCB2YWx1ZSwgYnkgY29udmVydGluZyB0aGVtIGZyb20gZGF0ZSBpbnRvIFVuaXggVGltZS4KVGhpcyB1bml0IG1lYXN1cmVzIHRpbWUgYnkgdGhlIG51bWJlciBvZiBzZWNvbmRzIHRoYXQgaGF2ZSBlbGFwc2VkIHNpbmNlIDAwOjAwOjAwIFVUQyBvbiAxIEphbnVhcnkgMTk3MC4KU2luY2UgdGhpcyBjb2x1bW4gZG9lc24ndCBjb250YWluIHRoZSBkYXkgbnVtYmVyLCB3ZSB0YWtlIGFzIGEgcmVmZXJlbmNlIHRoZSBmaXJzdCBkYXkgb2YgdGhlIG1vbnRoLgoKYGBge3J9CmxjX2RhdGEgPC0gbGNfZGF0YSAlPiUgCiAgICBmaWx0ZXIoIShpcy5uYShlYXJsaWVzdF9jcl9saW5lKSkpCgojIGZ1bmN0aW9uIHRvIHJlcGxhY2UgZGF0ZXMgd2l0aCB1bml4IHRpbWUKdG9fdW5peF90aW1lIDwtIGZ1bmN0aW9uKGRhdGUpIHsKICB0bXAgPC0gcGFzdGUoIjAxIiwgZGF0ZSwgc2VwPSItIikKICByZXR1cm4gKGFzLm51bWVyaWMoYXMuUE9TSVhjdCh0bXAsIGZvcm1hdD0iJWQtJWItJVkiLCB0ej0iVVRDIikpKQp9CgojIG1hcCBkYXRlcyB0byB1bml4IHRpbWUKbGNfZGF0YSRlYXJsaWVzdF9jcl9saW5lIDwtIGFwcGx5KGxjX2RhdGEsIDEsIGZ1bmN0aW9uKHJvdykgdG9fdW5peF90aW1lKHJvd1siZWFybGllc3RfY3JfbGluZSJdKSkKCiMgc3RhbmRhcmRpemUgdGhlbQojbGNfZGF0YSRlYXJsaWVzdF9jcl9saW5lIDwtIHNjYWxlKGxjX2RhdGEkZWFybGllc3RfY3JfbGluZSkKYGBgCgpPdXRsaWVycyBSZW1vdmFsOgoKYGBge3J9CmJveHBsb3QobGNfZGF0YSRpbnRfcmF0ZSkKIyBJZGVudGlmeSBvdXRsaWVycyB1c2luZyBib3hwbG90Cm91dGxpZXJzIDwtIGJveHBsb3QobGNfZGF0YSRpbnRfcmF0ZSwgcGxvdCA9IEZBTFNFKSRvdXQKIyBSZW1vdmUgb3V0bGllcnMgZnJvbSB0aGUgZGF0YXNldApsY19kYXRhIDwtIGxjX2RhdGFbIWxjX2RhdGEkaW50X3JhdGUgJWluJSBvdXRsaWVycywgXQpgYGAKCmBgYHtyfQpzdW1tYXJ5KGxjX2RhdGEpCmBgYAoKKipMZWFybmluZyBBbGdvcml0aG1zKioKCmBgYHtyfQojIGNyZWF0ZSBpbmRpY2VzIGZvciBzcGxpdHRpbmcgKDgwJSB0cmFpbiwgMjAlIHRlc3QpCnRyYWluX2luZGljZXMgPC0gY3JlYXRlRGF0YVBhcnRpdGlvbihsY19kYXRhJGludF9yYXRlLCBwID0gMC44LCBsaXN0ID0gRkFMU0UpCgojIGNyZWF0ZSB0cmFpbmluZyBhbmQgdGVzdGluZyBkYXRhc2V0cwp0cmFpbl9kYXRhIDwtIGxjX2RhdGFbdHJhaW5faW5kaWNlcywgXQp0ZXN0X2RhdGEgPC0gbGNfZGF0YVstdHJhaW5faW5kaWNlcywgXQpgYGAKCmBgYHtyfQojIyMjIExpbmVhciBSZWdyZXNzaW9uICMjIyMKCmxtLmZpdCA8LSBsbShpbnRfcmF0ZSB+IC4sIGRhdGEgPSB0cmFpbl9kYXRhKQoKIyBtYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0cmFpbmluZyBhbmQgdGVzdGluZyBkYXRhCmxtLnRyYWluX3ByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0uZml0LCBuZXdkYXRhID0gdHJhaW5fZGF0YSkKbG0udGVzdF9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGxtLmZpdCwgbmV3ZGF0YSA9IHRlc3RfZGF0YSkKCiMgY2FsY3VsYXRlIE1lYW4gU3F1YXJlZCBFcnJvciAoTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKbG0udHJhaW5fbXNlIDwtIG1lYW4oKGxtLnRyYWluX3ByZWRpY3Rpb25zIC0gdHJhaW5fZGF0YSRpbnRfcmF0ZSleMikKbG0udGVzdF9tc2UgPC0gbWVhbigobG0udGVzdF9wcmVkaWN0aW9ucyAtIHRlc3RfZGF0YSRpbnRfcmF0ZSleMikKCiMgY2FsY3VsYXRlIFJvb3QgTWVhbiBTcXVhcmVkIEVycm9yIChSTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKbG0udHJhaW5fcm1zZSA8LSBzcXJ0KGxtLnRyYWluX21zZSkKbG0udGVzdF9ybXNlIDwtIHNxcnQobG0udGVzdF9tc2UpCgojIGNhbGN1bGF0ZSBNZWFuIEFic29sdXRlIEVycm9yIChNQUUpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwpsbS50cmFpbl9tYWUgPC0gbWVhbihhYnMobG0udHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhJGludF9yYXRlKSkKbG0udGVzdF9tYWUgPC0gbWVhbihhYnMobG0udGVzdF9wcmVkaWN0aW9ucyAtIHRlc3RfZGF0YSRpbnRfcmF0ZSkpCgojIGNhbGN1bGF0ZSBSLXNxdWFyZWQgKFLCsikgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCmxtLnRyYWluX3IyIDwtIDEgLSAoc3VtKCh0cmFpbl9kYXRhJGludF9yYXRlIC0gbG0udHJhaW5fcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0cmFpbl9kYXRhJGludF9yYXRlIC0gbWVhbih0cmFpbl9kYXRhJGludF9yYXRlKSleMikpCmxtLnRlc3RfcjIgPC0gMSAtIChzdW0oKHRlc3RfZGF0YSRpbnRfcmF0ZSAtIGxtLnRlc3RfcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0ZXN0X2RhdGEkaW50X3JhdGUgLSBtZWFuKHRlc3RfZGF0YSRpbnRfcmF0ZSkpXjIpKQoKIyBkaXNwbGF5IHRoZSBtZXRyaWNzCmNhdCgiVHJhaW5pbmcgTVNFOiIsIGxtLnRyYWluX21zZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1TRToiLCBsbS50ZXN0X21zZSwgIlxuIikKY2F0KCJUcmFpbmluZyBSTVNFOiIsIGxtLnRyYWluX3Jtc2UsICJcbiIpCmNhdCgiVGVzdGluZyBSTVNFOiIsIGxtLnRlc3Rfcm1zZSwgIlxuIikKY2F0KCJUcmFpbmluZyBNQUU6IiwgbG0udHJhaW5fbWFlLCAiXG4iKQpjYXQoIlRlc3RpbmcgTUFFOiIsIGxtLnRlc3RfbWFlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFItc3F1YXJlZCAoUsKyKToiLCBsbS50cmFpbl9yMiwgIlxuIikKY2F0KCJUZXN0aW5nIFItc3F1YXJlZCAoUsKyKToiLCBsbS50ZXN0X3IyLCAiXG4iKQpgYGAKKipMYXNzbyoqCkl0IHN0YW5kYXJkaXplcyBkYXRhIGF1dG9tYXRpY2FsbHkKCmBgYHtyfQpsYXNzby5wcmVkaWN0b3JzX3RyYWluIDwtIG1vZGVsLm1hdHJpeChpbnRfcmF0ZSB+IC4sIHRyYWluX2RhdGEpWywtMV0KbGFzc28udGFyZ2V0X3RyYWluIDwtIHRyYWluX2RhdGEkaW50X3JhdGUKbGFzc28ucHJlZGljdG9yc190ZXN0IDwtIG1vZGVsLm1hdHJpeChpbnRfcmF0ZSB+IC4sIHRlc3RfZGF0YSlbLC0xXQpsYXNzby50YXJnZXRfdGVzdCA8LSB0ZXN0X2RhdGEkaW50X3JhdGUKCmxhc3NvLmZpdCA8LSBnbG1uZXQobGFzc28ucHJlZGljdG9yc190cmFpbiwgbGFzc28udGFyZ2V0X3RyYWluLCBhbHBoYSA9IDEpCgpwbG90KGxhc3NvLmZpdCwgbGFiZWw9VFJVRSkKCiMgbWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdHJhaW5pbmcgYW5kIHRlc3RpbmcgZGF0YQpsYXNzby50cmFpbl9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGxhc3NvLmZpdCwgbmV3ZGF0YSA9IHRyYWluX2RhdGEsIG5ld3ggPSBsYXNzby5wcmVkaWN0b3JzX3RyYWluKQpsYXNzby50ZXN0X3ByZWRpY3Rpb25zIDwtIHByZWRpY3QobGFzc28uZml0LCBuZXdkYXRhID0gdGVzdF9kYXRhLCBuZXd4ID0gbGFzc28ucHJlZGljdG9yc190cmFpbikKCiMgY2FsY3VsYXRlIE1lYW4gU3F1YXJlZCBFcnJvciAoTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKbGFzc28udHJhaW5fbXNlIDwtIG1lYW4oKGxhc3NvLnRyYWluX3ByZWRpY3Rpb25zIC0gdHJhaW5fZGF0YSRpbnRfcmF0ZSleMikKbGFzc28udGVzdF9tc2UgPC0gbWVhbigobGFzc28udGVzdF9wcmVkaWN0aW9ucyAtIHRlc3RfZGF0YSRpbnRfcmF0ZSleMikKCiMgY2FsY3VsYXRlIFJvb3QgTWVhbiBTcXVhcmVkIEVycm9yIChSTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKbGFzc28udHJhaW5fcm1zZSA8LSBzcXJ0KGxhc3NvLnRyYWluX21zZSkKbGFzc28udGVzdF9ybXNlIDwtIHNxcnQobGFzc28udGVzdF9tc2UpCgojIGNhbGN1bGF0ZSBNZWFuIEFic29sdXRlIEVycm9yIChNQUUpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwpsYXNzby50cmFpbl9tYWUgPC0gbWVhbihhYnMobGFzc28udHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhJGludF9yYXRlKSkKbGFzc28udGVzdF9tYWUgPC0gbWVhbihhYnMobGFzc28udGVzdF9wcmVkaWN0aW9ucyAtIHRlc3RfZGF0YSRpbnRfcmF0ZSkpCgojIGNhbGN1bGF0ZSBSLXNxdWFyZWQgKFLCsikgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCmxhc3NvLnRyYWluX3IyIDwtIDEgLSAoc3VtKCh0cmFpbl9kYXRhJGludF9yYXRlIC0gbGFzc28udHJhaW5fcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0cmFpbl9kYXRhJGludF9yYXRlIC0gbWVhbih0cmFpbl9kYXRhJGludF9yYXRlKSleMikpCmxhc3NvLnRlc3RfcjIgPC0gMSAtIChzdW0oKHRlc3RfZGF0YSRpbnRfcmF0ZSAtIGxhc3NvLnRlc3RfcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0ZXN0X2RhdGEkaW50X3JhdGUgLSBtZWFuKHRlc3RfZGF0YSRpbnRfcmF0ZSkpXjIpKQoKIyBkaXNwbGF5IHRoZSBtZXRyaWNzCmNhdCgiVHJhaW5pbmcgTVNFOiIsIGxhc3NvLnRyYWluX21zZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1TRToiLCBsYXNzby50ZXN0X21zZSwgIlxuIikKY2F0KCJUcmFpbmluZyBSTVNFOiIsIGxhc3NvLnRyYWluX3Jtc2UsICJcbiIpCmNhdCgiVGVzdGluZyBSTVNFOiIsIGxhc3NvLnRlc3Rfcm1zZSwgIlxuIikKY2F0KCJUcmFpbmluZyBNQUU6IiwgbGFzc28udHJhaW5fbWFlLCAiXG4iKQpjYXQoIlRlc3RpbmcgTUFFOiIsIGxhc3NvLnRlc3RfbWFlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFItc3F1YXJlZCAoUsKyKToiLCBsYXNzby50cmFpbl9yMiwgIlxuIikKY2F0KCJUZXN0aW5nIFItc3F1YXJlZCAoUsKyKToiLCBsYXNzby50ZXN0X3IyLCAiXG4iKQpgYGAKCioqUmlkZ2UqKgpJdCBzdGFuZGFyZGl6ZXMgZGF0YSBhdXRvbWF0aWNhbGx5CgpgYGB7cn0KcmlkZ2UucHJlZGljdG9yc190cmFpbiA8LSBtb2RlbC5tYXRyaXgoaW50X3JhdGUgfiAuLCB0cmFpbl9kYXRhKVssLTFdCnJpZGdlLnRhcmdldF90cmFpbiA8LSB0cmFpbl9kYXRhJGludF9yYXRlCnJpZGdlLnByZWRpY3RvcnNfdGVzdCA8LSBtb2RlbC5tYXRyaXgoaW50X3JhdGUgfiAuLCB0ZXN0X2RhdGEpWywtMV0KcmlkZ2UudGFyZ2V0X3Rlc3QgPC0gdGVzdF9kYXRhJGludF9yYXRlCgpyaWRnZS5maXQgPC0gZ2xtbmV0KHJpZGdlLnByZWRpY3RvcnNfdHJhaW4sIHJpZGdlLnRhcmdldF90cmFpbiwgYWxwaGEgPSAwKQoKcGxvdChyaWRnZS5maXQsIGxhYmVsPVRSVUUsIHhsYWIgPSAiTDIgTm9ybSIpCgojIG1ha2UgcHJlZGljdGlvbnMgb24gdGhlIHRyYWluaW5nIGFuZCB0ZXN0aW5nIGRhdGEKcmlkZ2UudHJhaW5fcHJlZGljdGlvbnMgPC0gcHJlZGljdChyaWRnZS5maXQsIG5ld2RhdGEgPSB0cmFpbl9kYXRhLCBuZXd4ID0gcmlkZ2UucHJlZGljdG9yc190cmFpbikKcmlkZ2UudGVzdF9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KHJpZGdlLmZpdCwgbmV3ZGF0YSA9IHRlc3RfZGF0YSwgbmV3eCA9IHJpZGdlLnByZWRpY3RvcnNfdHJhaW4pCgojIGNhbGN1bGF0ZSBNZWFuIFNxdWFyZWQgRXJyb3IgKE1TRSkgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCnJpZGdlLnRyYWluX21zZSA8LSBtZWFuKChyaWRnZS50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGEkaW50X3JhdGUpXjIpCnJpZGdlLnRlc3RfbXNlIDwtIG1lYW4oKHJpZGdlLnRlc3RfcHJlZGljdGlvbnMgLSB0ZXN0X2RhdGEkaW50X3JhdGUpXjIpCgojIGNhbGN1bGF0ZSBSb290IE1lYW4gU3F1YXJlZCBFcnJvciAoUk1TRSkgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCnJpZGdlLnRyYWluX3Jtc2UgPC0gc3FydChyaWRnZS50cmFpbl9tc2UpCnJpZGdlLnRlc3Rfcm1zZSA8LSBzcXJ0KHJpZGdlLnRlc3RfbXNlKQoKIyBjYWxjdWxhdGUgTWVhbiBBYnNvbHV0ZSBFcnJvciAoTUFFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKcmlkZ2UudHJhaW5fbWFlIDwtIG1lYW4oYWJzKHJpZGdlLnRyYWluX3ByZWRpY3Rpb25zIC0gdHJhaW5fZGF0YSRpbnRfcmF0ZSkpCnJpZGdlLnRlc3RfbWFlIDwtIG1lYW4oYWJzKHJpZGdlLnRlc3RfcHJlZGljdGlvbnMgLSB0ZXN0X2RhdGEkaW50X3JhdGUpKQoKIyBjYWxjdWxhdGUgUi1zcXVhcmVkIChSwrIpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwpyaWRnZS50cmFpbl9yMiA8LSAxIC0gKHN1bSgodHJhaW5fZGF0YSRpbnRfcmF0ZSAtIHJpZGdlLnRyYWluX3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgodHJhaW5fZGF0YSRpbnRfcmF0ZSAtIG1lYW4odHJhaW5fZGF0YSRpbnRfcmF0ZSkpXjIpKQpyaWRnZS50ZXN0X3IyIDwtIDEgLSAoc3VtKCh0ZXN0X2RhdGEkaW50X3JhdGUgLSByaWRnZS50ZXN0X3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgodGVzdF9kYXRhJGludF9yYXRlIC0gbWVhbih0ZXN0X2RhdGEkaW50X3JhdGUpKV4yKSkKCiMgZGlzcGxheSB0aGUgbWV0cmljcwpjYXQoIlRyYWluaW5nIE1TRToiLCByaWRnZS50cmFpbl9tc2UsICJcbiIpCmNhdCgiVGVzdGluZyBNU0U6IiwgcmlkZ2UudGVzdF9tc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgUk1TRToiLCByaWRnZS50cmFpbl9ybXNlLCAiXG4iKQpjYXQoIlRlc3RpbmcgUk1TRToiLCByaWRnZS50ZXN0X3Jtc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgTUFFOiIsIHJpZGdlLnRyYWluX21hZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1BRToiLCByaWRnZS50ZXN0X21hZSwgIlxuIikKY2F0KCJUcmFpbmluZyBSLXNxdWFyZWQgKFLCsik6IiwgcmlkZ2UudHJhaW5fcjIsICJcbiIpCmNhdCgiVGVzdGluZyBSLXNxdWFyZWQgKFLCsik6IiwgcmlkZ2UudGVzdF9yMiwgIlxuIikKYGBgCgpLIGZvbGQgdXNpbmcgSz01OgpgYGB7cn0KIyBkZWZpbmUgdGhlIG51bWJlciBvZiBmb2xkcyBmb3IgY3Jvc3MtdmFsaWRhdGlvbgpudW1fZm9sZHMgPC0gNQpmb2xkcyA8LSBjcmVhdGVGb2xkcyh0cmFpbl9kYXRhJGludF9yYXRlLCBrID0gbnVtX2ZvbGRzLCBsaXN0ID0gVFJVRSkKYGBgCgoKSyBmb2xkIHVzaW5nIEs9NSBhbmQgbGluZWFyIHJlZ3Jlc3Npb246CmBgYHtyfQojIyMjIExpbmVhciBSZWdyZXNpb24gYXBwbHlpbmcgQ3Jvc3MgVmFsaWRhdGlvbiB3aXRoIGs9NSAgIyMjIwoKIyBpbml0aWFsaXplIGxpc3RzIHRvIHN0b3JlIG1vZGVscyBhbmQgdGhlaXIgcmVzdWx0cwpsbS5rNS5tb2RlbHMgPC0gbGlzdCgpCmxtLms1LnJlc3VsdHMgPC0gZGF0YS5mcmFtZSgpCgojIHBlcmZvcm0gay1mb2xkIGNyb3NzLXZhbGlkYXRpb24KZm9yKGkgaW4gc2VxX2Fsb25nKGZvbGRzKSkgewogICMgc3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdGluZyBmb3IgdGhlIGN1cnJlbnQgZm9sZAogIHRyYWluX2luZGljZXMgPC0gZm9sZHNbW2ldXQogIHRlc3RfaW5kaWNlcyA8LSBzZXRkaWZmKHNlcV9sZW4obnJvdyh0cmFpbl9kYXRhKSksIHRyYWluX2luZGljZXMpCiAgCiAgdHJhaW5fZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdHJhaW5faW5kaWNlcywgXQogIHRlc3RfZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdGVzdF9pbmRpY2VzLCBdCiAgCiAgIyBmaXQgdGhlIG1vZGVsIG9uIHRoZSB0cmFpbmluZyBmb2xkCiAgbG0uazUgPC0gbG0oaW50X3JhdGUgfiAuLCBkYXRhID0gdHJhaW5fZGF0YV9mb2xkKQogIGxtLms1Lm1vZGVsc1tbaV1dIDwtIGxtLms1ICAjIFN0b3JlIHRoZSBtb2RlbAogIAogICMgbWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdHJhaW5pbmcgYW5kIHRlc3RpbmcgZm9sZAogIGxtLms1LnRyYWluX3ByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0uazUsIG5ld2RhdGEgPSB0cmFpbl9kYXRhX2ZvbGQpCiAgbG0uazUudGVzdF9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGxtLms1LCBuZXdkYXRhID0gdGVzdF9kYXRhX2ZvbGQpCiAgCiAgIyBjYWxjdWxhdGUgbWV0cmljcyBmb3IgdHJhaW5pbmcgZm9sZAogIGxtLms1LnRyYWluX21zZSA8LSBtZWFuKChsbS5rNS50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGFfZm9sZCRpbnRfcmF0ZSleMikKICBsbS5rNS50cmFpbl9ybXNlIDwtIHNxcnQobG0uazUudHJhaW5fbXNlKQogIGxtLms1LnRyYWluX21hZSA8LSBtZWFuKGFicyhsbS5rNS50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGFfZm9sZCRpbnRfcmF0ZSkpCiAgbG0uazUudHJhaW5fcjIgPC0gc3VtbWFyeShsbS5rNSkkci5zcXVhcmVkCiAgCiAgIyBjYWxjdWxhdGUgbWV0cmljcyBmb3IgdGVzdGluZyBmb2xkCiAgbG0uazUudGVzdF9tc2UgPC0gbWVhbigobG0uazUudGVzdF9wcmVkaWN0aW9ucyAtIHRlc3RfZGF0YV9mb2xkJGludF9yYXRlKV4yKQogIGxtLms1LnRlc3Rfcm1zZSA8LSBzcXJ0KGxtLms1LnRlc3RfbXNlKQogIGxtLms1LnRlc3RfbWFlIDwtIG1lYW4oYWJzKGxtLms1LnRlc3RfcHJlZGljdGlvbnMgLSB0ZXN0X2RhdGFfZm9sZCRpbnRfcmF0ZSkpCiAgbG0uazUudGVzdF9yMiA8LSAxIC0gKHN1bSgodGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUgLSBsbS5rNS50ZXN0X3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgodGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUgLSBtZWFuKHRlc3RfZGF0YV9mb2xkJGludF9yYXRlKSleMikpCiAgCiAgIyBzdG9yZSBtZXRyaWNzIGluIHRoZSByZXN1bHRzIGRhdGFmcmFtZQogIGxtLms1LnJlc3VsdHMgPC0gcmJpbmQobG0uazUucmVzdWx0cywgZGF0YS5mcmFtZSgKICAgIEZvbGQgPSBpLAogICAgVHJhaW5fTVNFID0gbG0uazUudHJhaW5fbXNlLCBUZXN0X01TRSA9IGxtLms1LnRlc3RfbXNlLAogICAgVHJhaW5fUk1TRSA9IGxtLms1LnRyYWluX3Jtc2UsIFRlc3RfUk1TRSA9IGxtLms1LnRlc3Rfcm1zZSwKICAgIFRyYWluX01BRSA9IGxtLms1LnRyYWluX21hZSwgVGVzdF9NQUUgPSBsbS5rNS50ZXN0X21hZSwKICAgIFRyYWluX1IyID0gbG0uazUudHJhaW5fcjIsIFRlc3RfUjIgPSBsbS5rNS50ZXN0X3IyCiAgKSkKfQoKIyBkaXNwbGF5IHRoZSBtb2RlbHMgYW5kIHRoZWlyIG1ldHJpY3MKcHJpbnQobG0uazUubW9kZWxzKQpwcmludChsbS5rNS5yZXN1bHRzKQpgYGAKCmBgYHtyfQpwbG90X21ldHJpYyA8LSBmdW5jdGlvbihyZXN1bHRzX2xvbmcsIG1ldHJpYykgewogICAgIyBhZGp1c3QgdGhlIHZhcmlhYmxlIG5hbWVzIGJhc2VkIG9uIHRoZSBtZXRyaWMKICAgIHZhcmlhYmxlcyA8LSBpZiAobWV0cmljID09ICJPT0IiKSB7CiAgICAgICAgIk9PQl9FcnJvciIKICAgIH0gZWxzZSB7CiAgICAgICAgYyhwYXN0ZTAoJ1RyYWluXycsIG1ldHJpYyksIHBhc3RlMCgnVGVzdF8nLCBtZXRyaWMpKQogICAgfQogICAgdGl0bGUgPC0gaWYgKG1ldHJpYyA9PSAiT09CIikgewogICAgICAgIHBhc3RlMChtZXRyaWMsICcgcGVyIEZvbGQnKQogICAgfSBlbHNlIHsKICAgICAgICBwYXN0ZTAoJ1RyYWluIHZzIFRlc3QgJywgbWV0cmljLCAnIHBlciBGb2xkJykKICAgIH0KICAgIAogICAgZ2dwbG90KHJlc3VsdHNfbG9uZ1tyZXN1bHRzX2xvbmckdmFyaWFibGUgJWluJSB2YXJpYWJsZXMsIF0sCiAgICAgICAgICAgYWVzKHggPSBGb2xkLCB5ID0gdmFsdWUsIGNvbG9yID0gdmFyaWFibGUpKSArCiAgICBnZW9tX2xpbmUoKSArCiAgICBnZW9tX3BvaW50KCkgKwogICAgdGhlbWVfbWluaW1hbCgpICsKICAgIGxhYnModGl0bGUgPSB0aXRsZSwKICAgICAgICAgeCA9ICdGb2xkJywKICAgICAgICAgeSA9IG1ldHJpYykKfQpgYGAKCgpgYGB7cn0KIyByZXNoYXBlIGRhdGEgZm9yIHBsb3R0aW5nCmxtLms1LnJlc3VsdHNfbG9uZyA8LSBtZWx0KGxtLms1LnJlc3VsdHMsIGlkLnZhcnMgPSAnRm9sZCcpCgojIHBsb3QgZm9yIGVhY2ggbWV0cmljCnAxIDwtIHBsb3RfbWV0cmljKGxtLms1LnJlc3VsdHNfbG9uZywgJ01TRScpCnAyIDwtIHBsb3RfbWV0cmljKGxtLms1LnJlc3VsdHNfbG9uZywgJ1JNU0UnKQpwMyA8LSBwbG90X21ldHJpYyhsbS5rNS5yZXN1bHRzX2xvbmcsICdNQUUnKQpwNCA8LSBwbG90X21ldHJpYyhsbS5rNS5yZXN1bHRzX2xvbmcsICdSMicpCgojIGFycmFuZ2UgdGhlIHBsb3RzIGluIGEgMngyIGdyaWQKZ3JpZC5hcnJhbmdlKHAxLCBwMiwgcDMsIHA0LCBuY29sID0gMiwgbnJvdyA9IDIpCgpwbG90KHAxKQpwbG90KHAyKQpwbG90KHAzKQpwbG90KHA0KQpgYGAKCksgZm9sZCB1c2luZyBLPTUgYW5kIFJhbmRvbSBGb3Jlc3Q6CmBgYHtyfQojIyMjIFJhbmRvbSBGb3Jlc3QgYXBwbHlpbmcgQ3Jvc3MgVmFsaWRhdGlvbiB3aXRoIGs9NSAgIyMjIwoKIyBpbml0aWFsaXplIGxpc3RzIHRvIHN0b3JlIG1vZGVscyBhbmQgdGhlaXIgcmVzdWx0cwpyZi5rNS5tb2RlbHMgPC0gbGlzdCgpCnJmLms1LnJlc3VsdHMgPC0gZGF0YS5mcmFtZSgpCgojIHBlcmZvcm0gay1mb2xkIGNyb3NzLXZhbGlkYXRpb24KZm9yKGkgaW4gc2VxX2Fsb25nKGZvbGRzKSkgewogICMgc3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdGluZyBmb3IgdGhlIGN1cnJlbnQgZm9sZAogIHRyYWluX2luZGljZXMgPC0gZm9sZHNbW2ldXQogIHRlc3RfaW5kaWNlcyA8LSBzZXRkaWZmKHNlcV9sZW4obnJvdyh0cmFpbl9kYXRhKSksIHRyYWluX2luZGljZXMpCiAgCiAgdHJhaW5fZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdHJhaW5faW5kaWNlcywgXQogIHRlc3RfZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdGVzdF9pbmRpY2VzLCBdCiAgCiAgIyBmaXQgdGhlIG1vZGVsIG9uIHRoZSB0cmFpbmluZyBmb2xkCiAgcmYuazUgPC0gcmFuZ2VyKGZvcm11bGEgPSBpbnRfcmF0ZSB+IC4sIGRhdGEgPSB0cmFpbl9kYXRhX2ZvbGQsIG51bS50cmVlcyA9IDUwMCwgdmVyYm9zZT1UUlVFLCBpbXBvcnRhbmNlID0gImltcHVyaXR5Iiwgb29iLmVycm9yID0gVFJVRSkKICByZi5rNS5tb2RlbHNbW2ldXSA8LSByZi5rNSAgIyBTdG9yZSB0aGUgbW9kZWwKICAKICAjIG1ha2UgcHJlZGljdGlvbnMgb24gdGhlIHRyYWluaW5nIGFuZCB0ZXN0aW5nIGZvbGQKICByZi5rNS50cmFpbl9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KHJmLms1LCBkYXRhID0gdHJhaW5fZGF0YV9mb2xkKSRwcmVkaWN0aW9ucwogIHJmLms1LnRlc3RfcHJlZGljdGlvbnMgPC0gcHJlZGljdChyZi5rNSwgZGF0YSA9IHRlc3RfZGF0YV9mb2xkKSRwcmVkaWN0aW9ucwogIAogICMgY2FsY3VsYXRlIG1ldHJpY3MgZm9yIHRyYWluaW5nIGZvbGQKICByZi5rNS50cmFpbl9tc2UgPC0gbWVhbigocmYuazUudHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhX2ZvbGQkaW50X3JhdGUpXjIpCiAgcmYuazUudHJhaW5fcm1zZSA8LSBzcXJ0KHJmLms1LnRyYWluX21zZSkKICByZi5rNS50cmFpbl9tYWUgPC0gbWVhbihhYnMocmYuazUudHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhX2ZvbGQkaW50X3JhdGUpKQogIHJmLms1Lm9vYl9lcnJvciA8LSByZi5rNSRwcmVkaWN0aW9uLmVycm9yCiAgCiAgIyBjYWxjdWxhdGUgbWV0cmljcyBmb3IgdGVzdGluZyBmb2xkCiAgcmYuazUudGVzdF9tc2UgPC0gbWVhbigocmYuazUudGVzdF9wcmVkaWN0aW9ucyAtIHRlc3RfZGF0YV9mb2xkJGludF9yYXRlKV4yKQogIHJmLms1LnRlc3Rfcm1zZSA8LSBzcXJ0KHJmLms1LnRlc3RfbXNlKQogIHJmLms1LnRlc3RfbWFlIDwtIG1lYW4oYWJzKHJmLms1LnRlc3RfcHJlZGljdGlvbnMgLSB0ZXN0X2RhdGFfZm9sZCRpbnRfcmF0ZSkpCiAgcmYuazUudGVzdF9yMiA8LSAxIC0gKHN1bSgodGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUgLSByZi5rNS50ZXN0X3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgodGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUgLSBtZWFuKHRlc3RfZGF0YV9mb2xkJGludF9yYXRlKSleMikpCiAgCiAgIyBzdG9yZSBtZXRyaWNzIGluIHRoZSByZXN1bHRzIGRhdGFmcmFtZQogIHJmLms1LnJlc3VsdHMgPC0gcmJpbmQocmYuazUucmVzdWx0cywgZGF0YS5mcmFtZSgKICAgIEZvbGQgPSBpLAogICAgVHJhaW5fTVNFID0gcmYuazUudHJhaW5fbXNlLCBUZXN0X01TRSA9IHJmLms1LnRlc3RfbXNlLAogICAgVHJhaW5fUk1TRSA9IHJmLms1LnRyYWluX3Jtc2UsIFRlc3RfUk1TRSA9IHJmLms1LnRlc3Rfcm1zZSwKICAgIFRyYWluX01BRSA9IHJmLms1LnRyYWluX21hZSwgVGVzdF9NQUUgPSByZi5rNS50ZXN0X21hZSwKICAgIE9PQl9FcnJvciA9IHJmLms1Lm9vYl9lcnJvcgogICkpCn0KCiMgZGlzcGxheSB0aGUgbW9kZWxzIGFuZCB0aGVpciBtZXRyaWNzCnByaW50KHJmLms1Lm1vZGVscykKcHJpbnQocmYuazUucmVzdWx0cykKYGBgCgoKYGBge3J9CiMgcmVzaGFwZSBkYXRhIGZvciBwbG90dGluZwpyZi5rNS5yZXN1bHRzX2xvbmcgPC0gbWVsdChyZi5rNS5yZXN1bHRzLCBpZC52YXJzID0gJ0ZvbGQnKQoKIyBwbG90IGZvciBlYWNoIG1ldHJpYwpwMSA8LSBwbG90X21ldHJpYyhyZi5rNS5yZXN1bHRzX2xvbmcsICdNU0UnKQpwMiA8LSBwbG90X21ldHJpYyhyZi5rNS5yZXN1bHRzX2xvbmcsICdSTVNFJykKcDMgPC0gcGxvdF9tZXRyaWMocmYuazUucmVzdWx0c19sb25nLCAnTUFFJykKcDQgPC0gcGxvdF9tZXRyaWMocmYuazUucmVzdWx0c19sb25nLCAnT09CJykKCiMgYXJyYW5nZSB0aGUgcGxvdHMgaW4gYSAyeDIgZ3JpZApncmlkLmFycmFuZ2UocDEsIHAyLCBwMywgcDQsIG5jb2wgPSAyLCBucm93ID0gMikKCnBsb3QocDEpCnBsb3QocDIpCnBsb3QocDMpCnBsb3QocDQpCmBgYAoKSyBmb2xkIHVzaW5nIEs9NSBhbmQgQm9vc3Rpbmc6CmBgYHtyfQojIyMjIEJvb3N0aW5nIGFwcGx5aW5nIENyb3NzIFZhbGlkYXRpb24gd2l0aCBrPTUgICMjIyMKCiMgaW5pdGlhbGl6ZSBsaXN0cyB0byBzdG9yZSBtb2RlbHMgYW5kIHRoZWlyIHJlc3VsdHMKeGdiLms1Lm1vZGVscyA8LSBsaXN0KCkKeGdiLms1LnJlc3VsdHMgPC0gZGF0YS5mcmFtZSgpCgojIHBlcmZvcm0gay1mb2xkIGNyb3NzLXZhbGlkYXRpb24KZm9yKGkgaW4gc2VxX2Fsb25nKGZvbGRzKSkgewogICMgc3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdGluZyBmb3IgdGhlIGN1cnJlbnQgZm9sZAogIHRyYWluX2luZGljZXMgPC0gZm9sZHNbW2ldXQogIHRlc3RfaW5kaWNlcyA8LSBzZXRkaWZmKHNlcV9sZW4obnJvdyh0cmFpbl9kYXRhKSksIHRyYWluX2luZGljZXMpCiAgCiAgdHJhaW5fZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdHJhaW5faW5kaWNlcywgXQogIHRlc3RfZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdGVzdF9pbmRpY2VzLCBdCiAgCiAgIyBwcmVwYXJlIGRhdGEgZm9yIHhnYm9vc3QKICB4Z2IueV90cmFpbl9mb2xkIDwtIHRyYWluX2RhdGFfZm9sZCRpbnRfcmF0ZQogIHhnYi5YX3RyYWluX2ZvbGQgPC0gYXMubWF0cml4KHRyYWluX2RhdGFfZm9sZFssIC13aGljaChuYW1lcyh0cmFpbl9kYXRhX2ZvbGQpID09ICdpbnRfcmF0ZScpXSkKICAKICB4Z2IueV90ZXN0X2ZvbGQgPC0gdGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUKICB4Z2IuWF90ZXN0X2ZvbGQgPC0gYXMubWF0cml4KHRlc3RfZGF0YV9mb2xkWywgLXdoaWNoKG5hbWVzKHRlc3RfZGF0YV9mb2xkKSA9PSAnaW50X3JhdGUnKV0pCiAgCiAgIyBmaXQgdGhlIHhnYm9vc3QgbW9kZWwgb24gdGhlIHRyYWluaW5nIGZvbGQKICB4Z2IuazUgPC0geGdib29zdCgKICAgIGRhdGEgPSB4Z2IuWF90cmFpbl9mb2xkLAogICAgbGFiZWwgPSB4Z2IueV90cmFpbl9mb2xkLAogICAgbnJvdW5kcyA9IDEwMCwKICAgIHZlcmJvc2UgPSAwCiAgKQogIHhnYi5rNS5tb2RlbHNbW2ldXSA8LSB4Z2IuazUgICMgc3RvcmUgdGhlIG1vZGVsCiAgCiAgIyBtYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0cmFpbmluZyBmb2xkCiAgeGdiLms1LnRyYWluX3ByZWRpY3Rpb25zIDwtIHByZWRpY3QoeGdiLms1LCBuZXdkYXRhID0geGdiLlhfdHJhaW5fZm9sZCkKICAjIG1ha2UgcHJlZGljdGlvbnMgb24gdGhlIHRlc3RpbmcgZm9sZAogIHhnYi5rNS50ZXN0X3ByZWRpY3Rpb25zIDwtIHByZWRpY3QoeGdiLms1LCBuZXdkYXRhID0geGdiLlhfdGVzdF9mb2xkKQogIAogICMgY2FsY3VsYXRlIG1ldHJpY3MgZm9yIHRyYWluaW5nIGZvbGQKICB4Z2IuazUudHJhaW5fbXNlIDwtIG1lYW4oKHhnYi5rNS50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGFfZm9sZCRpbnRfcmF0ZSleMikKICB4Z2IuazUudHJhaW5fcm1zZSA8LSBzcXJ0KHhnYi5rNS50cmFpbl9tc2UpCiAgeGdiLms1LnRyYWluX21hZSA8LSBtZWFuKGFicyh4Z2IuazUudHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhX2ZvbGQkaW50X3JhdGUpKQogIHhnYi5rNS50cmFpbl9yMiA8LSAxIC0gKHN1bSgoeGdiLnlfdHJhaW5fZm9sZCAtIHhnYi5rNS50cmFpbl9wcmVkaWN0aW9ucyleMikgLyBzdW0oKHhnYi55X3RyYWluX2ZvbGQgLSBtZWFuKHhnYi55X3RyYWluX2ZvbGQpKV4yKSkKCiAgIyBjYWxjdWxhdGUgbWV0cmljcyBmb3IgdGVzdGluZyBmb2xkCiAgeGdiLms1LnRlc3RfbXNlIDwtIG1lYW4oKHhnYi5rNS50ZXN0X3ByZWRpY3Rpb25zIC0geGdiLnlfdGVzdF9mb2xkKV4yKQogIHhnYi5rNS50ZXN0X3Jtc2UgPC0gc3FydCh4Z2IuazUudGVzdF9tc2UpCiAgeGdiLms1LnRlc3RfbWFlIDwtIG1lYW4oYWJzKHhnYi5rNS50ZXN0X3ByZWRpY3Rpb25zIC0geGdiLnlfdGVzdF9mb2xkKSkKICB4Z2IuazUudGVzdF9yMiA8LSAxIC0gKHN1bSgoeGdiLnlfdGVzdF9mb2xkIC0geGdiLms1LnRlc3RfcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh4Z2IueV90ZXN0X2ZvbGQgLSBtZWFuKHhnYi55X3Rlc3RfZm9sZCkpXjIpKSAgCiAgCiAgIyBzdG9yZSBtZXRyaWNzIGluIHRoZSByZXN1bHRzIGRhdGFmcmFtZQogIHhnYi5rNS5yZXN1bHRzIDwtIHJiaW5kKHhnYi5rNS5yZXN1bHRzLCBkYXRhLmZyYW1lKAogICAgRm9sZCA9IGksCiAgICBUcmFpbl9NU0UgPSB4Z2IuazUudHJhaW5fbXNlLCBUZXN0X01TRSA9IHhnYi5rNS50ZXN0X21zZSwKICAgIFRyYWluX1JNU0UgPSB4Z2IuazUudHJhaW5fcm1zZSwgVGVzdF9STVNFID0geGdiLms1LnRlc3Rfcm1zZSwKICAgIFRyYWluX01BRSA9IHhnYi5rNS50cmFpbl9tYWUsIFRlc3RfTUFFID0geGdiLms1LnRlc3RfbWFlLAogICAgVHJhaW5fUjIgPSB4Z2IuazUudHJhaW5fcjIsIFRlc3RfUjIgPSB4Z2IuazUudGVzdF9yMgogICkpCn0KCiMgZGlzcGxheSB0aGUgbW9kZWxzIGFuZCB0aGVpciBtZXRyaWNzCnByaW50KHhnYi5rNS5tb2RlbHMpCnByaW50KHhnYi5rNS5yZXN1bHRzKQpgYGAKYGBge3J9CiMgcmVzaGFwZSBkYXRhIGZvciBwbG90dGluZwp4Z2IuazUucmVzdWx0c19sb25nIDwtIG1lbHQoeGdiLms1LnJlc3VsdHMsIGlkLnZhcnMgPSAnRm9sZCcpCgojIHBsb3QgZm9yIGVhY2ggbWV0cmljCnAxIDwtIHBsb3RfbWV0cmljKHhnYi5rNS5yZXN1bHRzX2xvbmcsICdNU0UnKQpwMiA8LSBwbG90X21ldHJpYyh4Z2IuazUucmVzdWx0c19sb25nLCAnUk1TRScpCnAzIDwtIHBsb3RfbWV0cmljKHhnYi5rNS5yZXN1bHRzX2xvbmcsICdNQUUnKQpwNCA8LSBwbG90X21ldHJpYyh4Z2IuazUucmVzdWx0c19sb25nLCAnUjInKQoKIyBhcnJhbmdlIHRoZSBwbG90cyBpbiBhIDJ4MiBncmlkCmdyaWQuYXJyYW5nZShwMSwgcDIsIHAzLCBwNCwgbmNvbCA9IDIsIG5yb3cgPSAyKQoKcGxvdChwMSkKcGxvdChwMikKcGxvdChwMykKcGxvdChwNCkKYGBgCgpLIGZvbGQgdXNpbmcgSz0xMDoKYGBge3J9CiMgZGVmaW5lIHRoZSBudW1iZXIgb2YgZm9sZHMgZm9yIGNyb3NzLXZhbGlkYXRpb24KbnVtX2ZvbGRzIDwtIDEwCmZvbGRzIDwtIGNyZWF0ZUZvbGRzKHRyYWluX2RhdGEkaW50X3JhdGUsIGsgPSBudW1fZm9sZHMsIGxpc3QgPSBUUlVFKQpgYGAKCgpLIGZvbGQgdXNpbmcgSz0xMCBhbmQgbGluZWFyIHJlZ3Jlc3Npb246CmBgYHtyfQojIyMjIExpbmVhciBSZWdyZXNpb24gYXBwbHlpbmcgQ3Jvc3MgVmFsaWRhdGlvbiB3aXRoIGs9MTAgICMjIyMKCiMgaW5pdGlhbGl6ZSBsaXN0cyB0byBzdG9yZSBtb2RlbHMgYW5kIHRoZWlyIHJlc3VsdHMKbG0uazEwLm1vZGVscyA8LSBsaXN0KCkKbG0uazEwLnJlc3VsdHMgPC0gZGF0YS5mcmFtZSgpCgojIHBlcmZvcm0gay1mb2xkIGNyb3NzLXZhbGlkYXRpb24KZm9yKGkgaW4gc2VxX2Fsb25nKGZvbGRzKSkgewogICMgc3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdGluZyBmb3IgdGhlIGN1cnJlbnQgZm9sZAogIHRyYWluX2luZGljZXMgPC0gZm9sZHNbW2ldXQogIHRlc3RfaW5kaWNlcyA8LSBzZXRkaWZmKHNlcV9sZW4obnJvdyh0cmFpbl9kYXRhKSksIHRyYWluX2luZGljZXMpCiAgCiAgdHJhaW5fZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdHJhaW5faW5kaWNlcywgXQogIHRlc3RfZGF0YV9mb2xkIDwtIHRyYWluX2RhdGFbdGVzdF9pbmRpY2VzLCBdCiAgCiAgIyBmaXQgdGhlIG1vZGVsIG9uIHRoZSB0cmFpbmluZyBmb2xkCiAgbG0uazEwIDwtIGxtKGludF9yYXRlIH4gLiwgZGF0YSA9IHRyYWluX2RhdGFfZm9sZCkKICBsbS5rMTAubW9kZWxzW1tpXV0gPC0gbG0uazEwICAjIFN0b3JlIHRoZSBtb2RlbAogIAogICMgbWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdHJhaW5pbmcgYW5kIHRlc3RpbmcgZm9sZAogIGxtLmsxMC50cmFpbl9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGxtLmsxMCwgbmV3ZGF0YSA9IHRyYWluX2RhdGFfZm9sZCkKICBsbS5rMTAudGVzdF9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGxtLmsxMCwgbmV3ZGF0YSA9IHRlc3RfZGF0YV9mb2xkKQogIAogICMgY2FsY3VsYXRlIG1ldHJpY3MgZm9yIHRyYWluaW5nIGZvbGQKICBsbS5rMTAudHJhaW5fbXNlIDwtIG1lYW4oKGxtLmsxMC50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGFfZm9sZCRpbnRfcmF0ZSleMikKICBsbS5rMTAudHJhaW5fcm1zZSA8LSBzcXJ0KGxtLmsxMC50cmFpbl9tc2UpCiAgbG0uazEwLnRyYWluX21hZSA8LSBtZWFuKGFicyhsbS5rMTAudHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhX2ZvbGQkaW50X3JhdGUpKQogIGxtLmsxMC50cmFpbl9yMiA8LSBzdW1tYXJ5KGxtLmsxMCkkci5zcXVhcmVkCiAgCiAgIyBjYWxjdWxhdGUgbWV0cmljcyBmb3IgdGVzdGluZyBmb2xkCiAgbG0uazEwLnRlc3RfbXNlIDwtIG1lYW4oKGxtLmsxMC50ZXN0X3ByZWRpY3Rpb25zIC0gdGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUpXjIpCiAgbG0uazEwLnRlc3Rfcm1zZSA8LSBzcXJ0KGxtLmsxMC50ZXN0X21zZSkKICBsbS5rMTAudGVzdF9tYWUgPC0gbWVhbihhYnMobG0uazEwLnRlc3RfcHJlZGljdGlvbnMgLSB0ZXN0X2RhdGFfZm9sZCRpbnRfcmF0ZSkpCiAgbG0uazEwLnRlc3RfcjIgPC0gMSAtIChzdW0oKHRlc3RfZGF0YV9mb2xkJGludF9yYXRlIC0gbG0uazEwLnRlc3RfcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0ZXN0X2RhdGFfZm9sZCRpbnRfcmF0ZSAtIG1lYW4odGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUpKV4yKSkKICAKICAjIHN0b3JlIG1ldHJpY3MgaW4gdGhlIHJlc3VsdHMgZGF0YWZyYW1lCiAgbG0uazEwLnJlc3VsdHMgPC0gcmJpbmQobG0uazEwLnJlc3VsdHMsIGRhdGEuZnJhbWUoCiAgICBGb2xkID0gaSwKICAgIFRyYWluX01TRSA9IGxtLmsxMC50cmFpbl9tc2UsIFRlc3RfTVNFID0gbG0uazEwLnRlc3RfbXNlLAogICAgVHJhaW5fUk1TRSA9IGxtLmsxMC50cmFpbl9ybXNlLCBUZXN0X1JNU0UgPSBsbS5rMTAudGVzdF9ybXNlLAogICAgVHJhaW5fTUFFID0gbG0uazEwLnRyYWluX21hZSwgVGVzdF9NQUUgPSBsbS5rMTAudGVzdF9tYWUsCiAgICBUcmFpbl9SMiA9IGxtLmsxMC50cmFpbl9yMiwgVGVzdF9SMiA9IGxtLmsxMC50ZXN0X3IyCiAgKSkKfQoKIyBkaXNwbGF5IHRoZSBtb2RlbHMgYW5kIHRoZWlyIG1ldHJpY3MKcHJpbnQobG0uazEwLm1vZGVscykKcHJpbnQobG0uazEwLnJlc3VsdHMpCmBgYAoKYGBge3J9CnBsb3RfbWV0cmljIDwtIGZ1bmN0aW9uKHJlc3VsdHNfbG9uZywgbWV0cmljKSB7CiAgICAjIGFkanVzdCB0aGUgdmFyaWFibGUgbmFtZXMgYmFzZWQgb24gdGhlIG1ldHJpYwogICAgdmFyaWFibGVzIDwtIGlmIChtZXRyaWMgPT0gIk9PQiIpIHsKICAgICAgICAiT09CX0Vycm9yIgogICAgfSBlbHNlIHsKICAgICAgICBjKHBhc3RlMCgnVHJhaW5fJywgbWV0cmljKSwgcGFzdGUwKCdUZXN0XycsIG1ldHJpYykpCiAgICB9CiAgICB0aXRsZSA8LSBpZiAobWV0cmljID09ICJPT0IiKSB7CiAgICAgICAgcGFzdGUwKG1ldHJpYywgJyBwZXIgRm9sZCcpCiAgICB9IGVsc2UgewogICAgICAgIHBhc3RlMCgnVHJhaW4gdnMgVGVzdCAnLCBtZXRyaWMsICcgcGVyIEZvbGQnKQogICAgfQogICAgCiAgICBnZ3Bsb3QocmVzdWx0c19sb25nW3Jlc3VsdHNfbG9uZyR2YXJpYWJsZSAlaW4lIHZhcmlhYmxlcywgXSwKICAgICAgICAgICBhZXMoeCA9IEZvbGQsIHkgPSB2YWx1ZSwgY29sb3IgPSB2YXJpYWJsZSkpICsKICAgIGdlb21fbGluZSgpICsKICAgIGdlb21fcG9pbnQoKSArCiAgICB0aGVtZV9taW5pbWFsKCkgKwogICAgbGFicyh0aXRsZSA9IHRpdGxlLAogICAgICAgICB4ID0gJ0ZvbGQnLAogICAgICAgICB5ID0gbWV0cmljKQp9CmBgYAoKCmBgYHtyfQojIHJlc2hhcGUgZGF0YSBmb3IgcGxvdHRpbmcKbG0uazEwLnJlc3VsdHNfbG9uZyA8LSBtZWx0KGxtLmsxMC5yZXN1bHRzLCBpZC52YXJzID0gJ0ZvbGQnKQoKIyBwbG90IGZvciBlYWNoIG1ldHJpYwpwMSA8LSBwbG90X21ldHJpYyhsbS5rMTAucmVzdWx0c19sb25nLCAnTVNFJykKcDIgPC0gcGxvdF9tZXRyaWMobG0uazEwLnJlc3VsdHNfbG9uZywgJ1JNU0UnKQpwMyA8LSBwbG90X21ldHJpYyhsbS5rMTAucmVzdWx0c19sb25nLCAnTUFFJykKcDQgPC0gcGxvdF9tZXRyaWMobG0uazEwLnJlc3VsdHNfbG9uZywgJ1IyJykKCiMgYXJyYW5nZSB0aGUgcGxvdHMgaW4gYSAyeDIgZ3JpZApncmlkLmFycmFuZ2UocDEsIHAyLCBwMywgcDQsIG5jb2wgPSAyLCBucm93ID0gMikKCnBsb3QocDEpCnBsb3QocDIpCnBsb3QocDMpCnBsb3QocDQpCmBgYAoKSyBmb2xkIHVzaW5nIEs9MTAgYW5kIFJhbmRvbSBGb3Jlc3Q6CmBgYHtyfQojICMjIyMgUmFuZG9tIEZvcmVzdCBhcHBseWluZyBDcm9zcyBWYWxpZGF0aW9uIHdpdGggaz0xMCAgIyMjIwojIAojICMgaW5pdGlhbGl6ZSBsaXN0cyB0byBzdG9yZSBtb2RlbHMgYW5kIHRoZWlyIHJlc3VsdHMKIyByZi5rMTAubW9kZWxzIDwtIGxpc3QoKQojIHJmLmsxMC5yZXN1bHRzIDwtIGRhdGEuZnJhbWUoKQojIAojICMgcGVyZm9ybSBrLWZvbGQgY3Jvc3MtdmFsaWRhdGlvbgojIGZvcihpIGluIHNlcV9hbG9uZyhmb2xkcykpIHsKIyAgICMgc3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdGluZyBmb3IgdGhlIGN1cnJlbnQgZm9sZAojICAgdHJhaW5faW5kaWNlcyA8LSBmb2xkc1tbaV1dCiMgICB0ZXN0X2luZGljZXMgPC0gc2V0ZGlmZihzZXFfbGVuKG5yb3codHJhaW5fZGF0YSkpLCB0cmFpbl9pbmRpY2VzKQojICAgCiMgICB0cmFpbl9kYXRhX2ZvbGQgPC0gdHJhaW5fZGF0YVt0cmFpbl9pbmRpY2VzLCBdCiMgICB0ZXN0X2RhdGFfZm9sZCA8LSB0cmFpbl9kYXRhW3Rlc3RfaW5kaWNlcywgXQojICAgCiMgICAjIGZpdCB0aGUgbW9kZWwgb24gdGhlIHRyYWluaW5nIGZvbGQKIyAgIHJmLmsxMCA8LSByYW5nZXIoZm9ybXVsYSA9IGludF9yYXRlIH4gLiwgZGF0YSA9IHRyYWluX2RhdGEsIG51bS50cmVlcyA9IDUwMCwgdmVyYm9zZT1UUlVFLCBpbXBvcnRhbmNlID0gImltcHVyaXR5Iiwgb29iLmVycm9yID0gVFJVRSkKIyAgIHJmLmsxMC5tb2RlbHNbW2ldXSA8LSByZi5rMTAgICMgU3RvcmUgdGhlIG1vZGVsCiMgICAKIyAgICMgbWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdHJhaW5pbmcgYW5kIHRlc3RpbmcgZm9sZAojICAgcmYuazEwLnRyYWluX3ByZWRpY3Rpb25zIDwtIHByZWRpY3QocmYuazEwLCBkYXRhID0gdHJhaW5fZGF0YV9mb2xkKSRwcmVkaWN0aW9ucwojICAgcmYuazEwLnRlc3RfcHJlZGljdGlvbnMgPC0gcHJlZGljdChyZi5rMTAsIGRhdGEgPSB0ZXN0X2RhdGFfZm9sZCkkcHJlZGljdGlvbnMKIyAgIAojICAgIyBjYWxjdWxhdGUgbWV0cmljcyBmb3IgdHJhaW5pbmcgZm9sZAojICAgcmYuazEwLnRyYWluX21zZSA8LSBtZWFuKChyZi5rMTAudHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhX2ZvbGQkaW50X3JhdGUpXjIpCiMgICByZi5rMTAudHJhaW5fcm1zZSA8LSBzcXJ0KHJmLmsxMC50cmFpbl9tc2UpCiMgICByZi5rMTAudHJhaW5fbWFlIDwtIG1lYW4oYWJzKHJmLmsxMC50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGFfZm9sZCRpbnRfcmF0ZSkpCiMgICByZi5rMTAub29iX2Vycm9yIDwtIHJmLmsxMCRwcmVkaWN0aW9uLmVycm9yCiMgICAKIyAgICMgY2FsY3VsYXRlIG1ldHJpY3MgZm9yIHRlc3RpbmcgZm9sZAojICAgcmYuazEwLnRlc3RfbXNlIDwtIG1lYW4oKHJmLmsxMC50ZXN0X3ByZWRpY3Rpb25zIC0gdGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUpXjIpCiMgICByZi5rMTAudGVzdF9ybXNlIDwtIHNxcnQocmYuazEwLnRlc3RfbXNlKQojICAgcmYuazEwLnRlc3RfbWFlIDwtIG1lYW4oYWJzKHJmLmsxMC50ZXN0X3ByZWRpY3Rpb25zIC0gdGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUpKQojICAgcmYuazEwLnRlc3RfcjIgPC0gMSAtIChzdW0oKHRlc3RfZGF0YV9mb2xkJGludF9yYXRlIC0gcmYuazEwLnRlc3RfcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0ZXN0X2RhdGFfZm9sZCRpbnRfcmF0ZSAtIG1lYW4odGVzdF9kYXRhX2ZvbGQkaW50X3JhdGUpKV4yKSkKIyAgIAojICAgIyBzdG9yZSBtZXRyaWNzIGluIHRoZSByZXN1bHRzIGRhdGFmcmFtZQojICAgcmYuazEwLnJlc3VsdHMgPC0gcmJpbmQocmYuazEwLnJlc3VsdHMsIGRhdGEuZnJhbWUoCiMgICAgIEZvbGQgPSBpLAojICAgICBUcmFpbl9NU0UgPSByZi5rMTAudHJhaW5fbXNlLCBUZXN0X01TRSA9IHJmLmsxMC50ZXN0X21zZSwKIyAgICAgVHJhaW5fUk1TRSA9IHJmLmsxMC50cmFpbl9ybXNlLCBUZXN0X1JNU0UgPSByZi5rMTAudGVzdF9ybXNlLAojICAgICBUcmFpbl9NQUUgPSByZi5rMTAudHJhaW5fbWFlLCBUZXN0X01BRSA9IHJmLmsxMC50ZXN0X21hZSwKIyAgICAgT09CX0Vycm9yID0gcmYuazEwLm9vYl9lcnJvcgojICAgKSkKIyB9CiMgCiMgIyBkaXNwbGF5IHRoZSBtb2RlbHMgYW5kIHRoZWlyIG1ldHJpY3MKIyBwcmludChyZi5rMTAubW9kZWxzKQojIHByaW50KHJmLmsxMC5yZXN1bHRzKQpgYGAKYGBge3J9CiMgcmVzaGFwZSBkYXRhIGZvciBwbG90dGluZwojIHJmLmsxMC5yZXN1bHRzX2xvbmcgPC0gbWVsdChyZi5rMTAucmVzdWx0cywgaWQudmFycyA9ICdGb2xkJykKIyAKIyAjIHBsb3QgZm9yIGVhY2ggbWV0cmljCiMgcDEgPC0gcGxvdF9tZXRyaWMocmYuazEwLnJlc3VsdHNfbG9uZywgJ01TRScpCiMgcDIgPC0gcGxvdF9tZXRyaWMocmYuazEwLnJlc3VsdHNfbG9uZywgJ1JNU0UnKQojIHAzIDwtIHBsb3RfbWV0cmljKHJmLmsxMC5yZXN1bHRzX2xvbmcsICdNQUUnKQojIHA0IDwtIHBsb3RfbWV0cmljKHJmLmsxMC5yZXN1bHRzX2xvbmcsICdPT0InKQojIAojICMgYXJyYW5nZSB0aGUgcGxvdHMgaW4gYSAyeDIgZ3JpZAojIGdyaWQuYXJyYW5nZShwMSwgcDIsIHAzLCBwNCwgbmNvbCA9IDIsIG5yb3cgPSAyKQojIAojIHBsb3QocDEpCiMgcGxvdChwMikKIyBwbG90KHAzKQojIHBsb3QocDQpCmBgYAoKSyBmb2xkIHVzaW5nIEs9MTAgYW5kIEJvb3N0aW5nOgpgYGB7cn0KIyMjIyBCb29zdGluZyBhcHBseWluZyBDcm9zcyBWYWxpZGF0aW9uIHdpdGggaz0xMCAgIyMjIwoKIyBpbml0aWFsaXplIGxpc3RzIHRvIHN0b3JlIG1vZGVscyBhbmQgdGhlaXIgcmVzdWx0cwp4Z2IuazEwLm1vZGVscyA8LSBsaXN0KCkKeGdiLmsxMC5yZXN1bHRzIDwtIGRhdGEuZnJhbWUoKQoKIyBwZXJmb3JtIGstZm9sZCBjcm9zcy12YWxpZGF0aW9uCmZvcihpIGluIHNlcV9hbG9uZyhmb2xkcykpIHsKICAjIHNwbGl0IHRoZSBkYXRhIGludG8gdHJhaW5pbmcgYW5kIHRlc3RpbmcgZm9yIHRoZSBjdXJyZW50IGZvbGQKICB0cmFpbl9pbmRpY2VzIDwtIGZvbGRzW1tpXV0KICB0ZXN0X2luZGljZXMgPC0gc2V0ZGlmZihzZXFfbGVuKG5yb3codHJhaW5fZGF0YSkpLCB0cmFpbl9pbmRpY2VzKQogIAogIHRyYWluX2RhdGFfZm9sZCA8LSB0cmFpbl9kYXRhW3RyYWluX2luZGljZXMsIF0KICB0ZXN0X2RhdGFfZm9sZCA8LSB0cmFpbl9kYXRhW3Rlc3RfaW5kaWNlcywgXQogIAogICMgcHJlcGFyZSBkYXRhIGZvciB4Z2Jvb3N0CiAgeGdiLnlfdHJhaW5fZm9sZCA8LSB0cmFpbl9kYXRhX2ZvbGQkaW50X3JhdGUKICB4Z2IuWF90cmFpbl9mb2xkIDwtIGFzLm1hdHJpeCh0cmFpbl9kYXRhX2ZvbGRbLCAtd2hpY2gobmFtZXModHJhaW5fZGF0YV9mb2xkKSA9PSAnaW50X3JhdGUnKV0pCiAgCiAgeGdiLnlfdGVzdF9mb2xkIDwtIHRlc3RfZGF0YV9mb2xkJGludF9yYXRlCiAgeGdiLlhfdGVzdF9mb2xkIDwtIGFzLm1hdHJpeCh0ZXN0X2RhdGFfZm9sZFssIC13aGljaChuYW1lcyh0ZXN0X2RhdGFfZm9sZCkgPT0gJ2ludF9yYXRlJyldKQogIAogICMgZml0IHRoZSB4Z2Jvb3N0IG1vZGVsIG9uIHRoZSB0cmFpbmluZyBmb2xkCiAgeGdiLmsxMCA8LSB4Z2Jvb3N0KAogICAgZGF0YSA9IHhnYi5YX3RyYWluX2ZvbGQsCiAgICBsYWJlbCA9IHhnYi55X3RyYWluX2ZvbGQsCiAgICBucm91bmRzID0gMTAwLAogICAgdmVyYm9zZSA9IDAKICApCiAgeGdiLmsxMC5tb2RlbHNbW2ldXSA8LSB4Z2IuazEwICAjIHN0b3JlIHRoZSBtb2RlbAogIAogICMgbWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdHJhaW5pbmcgZm9sZAogIHhnYi5rMTAudHJhaW5fcHJlZGljdGlvbnMgPC0gcHJlZGljdCh4Z2IuazEwLCBuZXdkYXRhID0geGdiLlhfdHJhaW5fZm9sZCkKICAjIG1ha2UgcHJlZGljdGlvbnMgb24gdGhlIHRlc3RpbmcgZm9sZAogIHhnYi5rMTAudGVzdF9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KHhnYi5rMTAsIG5ld2RhdGEgPSB4Z2IuWF90ZXN0X2ZvbGQpCiAgCiAgIyBjYWxjdWxhdGUgbWV0cmljcyBmb3IgdHJhaW5pbmcgZm9sZAogIHhnYi5rMTAudHJhaW5fbXNlIDwtIG1lYW4oKHhnYi5rMTAudHJhaW5fcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhX2ZvbGQkaW50X3JhdGUpXjIpCiAgeGdiLmsxMC50cmFpbl9ybXNlIDwtIHNxcnQoeGdiLmsxMC50cmFpbl9tc2UpCiAgeGdiLmsxMC50cmFpbl9tYWUgPC0gbWVhbihhYnMoeGdiLmsxMC50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGFfZm9sZCRpbnRfcmF0ZSkpCiAgeGdiLmsxMC50cmFpbl9yMiA8LSAxIC0gKHN1bSgoeGdiLnlfdHJhaW5fZm9sZCAtIHhnYi5rMTAudHJhaW5fcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh4Z2IueV90cmFpbl9mb2xkIC0gbWVhbih4Z2IueV90cmFpbl9mb2xkKSleMikpCgogICMgY2FsY3VsYXRlIG1ldHJpY3MgZm9yIHRlc3RpbmcgZm9sZAogIHhnYi5rMTAudGVzdF9tc2UgPC0gbWVhbigoeGdiLmsxMC50ZXN0X3ByZWRpY3Rpb25zIC0geGdiLnlfdGVzdF9mb2xkKV4yKQogIHhnYi5rMTAudGVzdF9ybXNlIDwtIHNxcnQoeGdiLmsxMC50ZXN0X21zZSkKICB4Z2IuazEwLnRlc3RfbWFlIDwtIG1lYW4oYWJzKHhnYi5rMTAudGVzdF9wcmVkaWN0aW9ucyAtIHhnYi55X3Rlc3RfZm9sZCkpCiAgeGdiLmsxMC50ZXN0X3IyIDwtIDEgLSAoc3VtKCh4Z2IueV90ZXN0X2ZvbGQgLSB4Z2IuazEwLnRlc3RfcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh4Z2IueV90ZXN0X2ZvbGQgLSBtZWFuKHhnYi55X3Rlc3RfZm9sZCkpXjIpKSAgCiAgCiAgIyBzdG9yZSBtZXRyaWNzIGluIHRoZSByZXN1bHRzIGRhdGFmcmFtZQogIHhnYi5rMTAucmVzdWx0cyA8LSByYmluZCh4Z2IuazEwLnJlc3VsdHMsIGRhdGEuZnJhbWUoCiAgICBGb2xkID0gaSwKICAgIFRyYWluX01TRSA9IHhnYi5rMTAudHJhaW5fbXNlLCBUZXN0X01TRSA9IHhnYi5rMTAudGVzdF9tc2UsCiAgICBUcmFpbl9STVNFID0geGdiLmsxMC50cmFpbl9ybXNlLCBUZXN0X1JNU0UgPSB4Z2IuazEwLnRlc3Rfcm1zZSwKICAgIFRyYWluX01BRSA9IHhnYi5rMTAudHJhaW5fbWFlLCBUZXN0X01BRSA9IHhnYi5rMTAudGVzdF9tYWUsCiAgICBUcmFpbl9SMiA9IHhnYi5rMTAudHJhaW5fcjIsIFRlc3RfUjIgPSB4Z2IuazEwLnRlc3RfcjIKICApKQp9CgojIGRpc3BsYXkgdGhlIG1vZGVscyBhbmQgdGhlaXIgbWV0cmljcwpwcmludCh4Z2IuazEwLm1vZGVscykKcHJpbnQoeGdiLmsxMC5yZXN1bHRzKQpgYGAKYGBge3J9CiMgcmVzaGFwZSBkYXRhIGZvciBwbG90dGluZwp4Z2IuazEwLnJlc3VsdHNfbG9uZyA8LSBtZWx0KHhnYi5rMTAucmVzdWx0cywgaWQudmFycyA9ICdGb2xkJykKCiMgcGxvdCBmb3IgZWFjaCBtZXRyaWMKcDEgPC0gcGxvdF9tZXRyaWMoeGdiLmsxMC5yZXN1bHRzX2xvbmcsICdNU0UnKQpwMiA8LSBwbG90X21ldHJpYyh4Z2IuazEwLnJlc3VsdHNfbG9uZywgJ1JNU0UnKQpwMyA8LSBwbG90X21ldHJpYyh4Z2IuazEwLnJlc3VsdHNfbG9uZywgJ01BRScpCnA0IDwtIHBsb3RfbWV0cmljKHhnYi5rMTAucmVzdWx0c19sb25nLCAnUjInKQoKIyBhcnJhbmdlIHRoZSBwbG90cyBpbiBhIDJ4MiBncmlkCmdyaWQuYXJyYW5nZShwMSwgcDIsIHAzLCBwNCwgbmNvbCA9IDIsIG5yb3cgPSAyKQoKcGxvdChwMSkKcGxvdChwMikKcGxvdChwMykKcGxvdChwNCkKYGBgCgpEZWNpc2lvbiBUcmVlcwpgYGB7cn0KIyMjIyBEZWNpc2lvbiBUcmVlcyAjIyMjCgojIGVycm9yIGluIHRyZWU6ICJmYWN0b3IgcHJlZGljdG9ycyBtdXN0IGhhdmUgYXQgbW9zdCAzMiBsZXZlbHMiIGlzIHRocm93bgoKIyBiYXNpY2FsbHksIGl0IGJlY29tZXMgY29tcHV0YXRpb25hbGx5IGV4cGVuc2l2ZSB0byBjcmVhdGUgc28gbWFueSBzcGxpdHMgaW4geW91ciBkYXRhLCBzaW5jZSB5b3UgYXJlIHNlbGVjdGluZyB0aGUgYmVzdCBzcGxpdCBvdXQgb2YgYWxsIDJeMzIgKGFwcHJveCkgcG9zc2libGUgc3BsaXRzCgojIFRoZSBlcnJvciBhYm92ZSB3YXMgc29sdmVkIHdpdGggdGhlIGZhY3RvciBhbmQgdGhlbiBudW1lcmljIHZhcmlhYmxlIHRyYW5zZm9ybWF0aW9uCgojIGZpdCBhIGRlY2lzaW9uIHRyZWUgbW9kZWwgb24gdGhlIHRyYWluaW5nIGRhdGEKdG0gPC0gdHJlZShpbnRfcmF0ZSB+IC4sIGRhdGEgPSB0cmFpbl9kYXRhKQoKIyBtYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0cmFpbmluZyBhbmQgdGVzdGluZyBkYXRhCnRtLnRyYWluX3ByZWRpY3Rpb25zIDwtIHByZWRpY3QodG0sIG5ld2RhdGEgPSB0cmFpbl9kYXRhKQp0bS50ZXN0X3ByZWRpY3Rpb25zIDwtIHByZWRpY3QodG0sIG5ld2RhdGEgPSB0ZXN0X2RhdGEpCgojIGNhbGN1bGF0ZSBNZWFuIFNxdWFyZWQgRXJyb3IgKE1TRSkgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCnRtLnRyYWluX21zZSA8LSBtZWFuKCh0bS50cmFpbl9wcmVkaWN0aW9ucyAtIHRyYWluX2RhdGEkaW50X3JhdGUpXjIpCnRtLnRlc3RfbXNlIDwtIG1lYW4oKHRtLnRlc3RfcHJlZGljdGlvbnMgLSB0ZXN0X2RhdGEkaW50X3JhdGUpXjIpCgojIGNhbGN1bGF0ZSBSb290IE1lYW4gU3F1YXJlZCBFcnJvciAoUk1TRSkgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCnRtLnRyYWluX3Jtc2UgPC0gc3FydCh0bS50cmFpbl9tc2UpCnRtLnRlc3Rfcm1zZSA8LSBzcXJ0KHRtLnRlc3RfbXNlKQoKIyBjYWxjdWxhdGUgTWVhbiBBYnNvbHV0ZSBFcnJvciAoTUFFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKdG0udHJhaW5fbWFlIDwtIG1lYW4oYWJzKHRtLnRyYWluX3ByZWRpY3Rpb25zIC0gdHJhaW5fZGF0YSRpbnRfcmF0ZSkpCnRtLnRlc3RfbWFlIDwtIG1lYW4oYWJzKHRtLnRlc3RfcHJlZGljdGlvbnMgLSB0ZXN0X2RhdGEkaW50X3JhdGUpKQoKIyBjYWxjdWxhdGUgUi1zcXVhcmVkIChSwrIpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwp0bS50cmFpbl9yMiA8LSAxIC0gKHN1bSgodHJhaW5fZGF0YSRpbnRfcmF0ZSAtIHRtLnRyYWluX3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgodHJhaW5fZGF0YSRpbnRfcmF0ZSAtIG1lYW4odHJhaW5fZGF0YSRpbnRfcmF0ZSkpXjIpKQp0bS50ZXN0X3IyIDwtIDEgLSAoc3VtKCh0ZXN0X2RhdGEkaW50X3JhdGUgLSB0bS50ZXN0X3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgodGVzdF9kYXRhJGludF9yYXRlIC0gbWVhbih0ZXN0X2RhdGEkaW50X3JhdGUpKV4yKSkKCiMgZGlzcGxheSB0aGUgbWV0cmljcwpjYXQoIlRyYWluaW5nIE1TRToiLCB0bS50cmFpbl9tc2UsICJcbiIpCmNhdCgiVGVzdGluZyBNU0U6IiwgdG0udGVzdF9tc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgUk1TRToiLCB0bS50cmFpbl9ybXNlLCAiXG4iKQpjYXQoIlRlc3RpbmcgUk1TRToiLCB0bS50ZXN0X3Jtc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgTUFFOiIsIHRtLnRyYWluX21hZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1BRToiLCB0bS50ZXN0X21hZSwgIlxuIikKY2F0KCJUcmFpbmluZyBSLXNxdWFyZWQgKFLCsik6IiwgdG0udHJhaW5fcjIsICJcbiIpCmNhdCgiVGVzdGluZyBSLXNxdWFyZWQgKFLCsik6IiwgdG0udGVzdF9yMiwgIlxuIikKYGBgCgpSYW5kb20gRm9yZXN0CmBgYHtyfQojIyMjIFJhbmRvbSBGb3Jlc3QgIyMjIwoKIyB0cmFpbiBhIFJhbmRvbSBGb3Jlc3QgbW9kZWwKcmYgPC0gcmFuZ2VyKGZvcm11bGEgPSBpbnRfcmF0ZSB+IC4sIGRhdGEgPSB0cmFpbl9kYXRhLCBudW0udHJlZXMgPSA1MDAsIHZlcmJvc2U9VFJVRSwgaW1wb3J0YW5jZSA9ICJpbXB1cml0eSIsIG9vYi5lcnJvciA9IFRSVUUpCgojIHByaW50IHRoZSBtb2RlbCBzdW1tYXJ5CnByaW50KCJSYW5kb20gRm9yZXN0IE1vZGVsIFN1bW1hcnk6IikKcHJpbnQocmYpCgojIG1ha2UgcHJlZGljdGlvbnMgb24gdGhlIHRyYWluaW5nIGFuZCB0ZXN0aW5nIGRhdGEKcmYudHJhaW5fcHJlZGljdGlvbnMgPC0gcHJlZGljdChyZiwgZGF0YSA9IHRyYWluX2RhdGEpCnJmLnRlc3RfcHJlZGljdGlvbnMgPC0gcHJlZGljdChyZiwgZGF0YSA9IHRlc3RfZGF0YSkKCiMgY2FsY3VsYXRlIE1lYW4gU3F1YXJlZCBFcnJvciAoTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKcmYudHJhaW5fbXNlIDwtIG1lYW4oKHJmLnRyYWluX3ByZWRpY3Rpb25zJHByZWRpY3Rpb25zIC0gdHJhaW5fZGF0YSRpbnRfcmF0ZSleMikKcmYudGVzdF9tc2UgPC0gbWVhbigocmYudGVzdF9wcmVkaWN0aW9ucyRwcmVkaWN0aW9ucyAtIHRlc3RfZGF0YSRpbnRfcmF0ZSleMikKCiMgY2FsY3VsYXRlIFJvb3QgTWVhbiBTcXVhcmVkIEVycm9yIChSTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKcmYudHJhaW5fcm1zZSA8LSBzcXJ0KHJmLnRyYWluX21zZSkKcmYudGVzdF9ybXNlIDwtIHNxcnQocmYudGVzdF9tc2UpCgojIGNhbGN1bGF0ZSBNZWFuIEFic29sdXRlIEVycm9yIChNQUUpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwpyZi50cmFpbl9tYWUgPC0gbWVhbihhYnMocmYudHJhaW5fcHJlZGljdGlvbnMkcHJlZGljdGlvbnMgLSB0cmFpbl9kYXRhJGludF9yYXRlKSkKcmYudGVzdF9tYWUgPC0gbWVhbihhYnMocmYudGVzdF9wcmVkaWN0aW9ucyRwcmVkaWN0aW9ucyAtIHRlc3RfZGF0YSRpbnRfcmF0ZSkpCgojIGNhbGN1bGF0ZSBSLXNxdWFyZWQgKFLCsikgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCnJmLnRyYWluX3IyIDwtIDEgLSAoc3VtKCh0cmFpbl9kYXRhJGludF9yYXRlIC0gcmYudHJhaW5fcHJlZGljdGlvbnMkcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0cmFpbl9kYXRhJGludF9yYXRlIC0gbWVhbih0cmFpbl9kYXRhJGludF9yYXRlKSleMikpCnJmLnRlc3RfcjIgPC0gMSAtIChzdW0oKHRlc3RfZGF0YSRpbnRfcmF0ZSAtIHJmLnRlc3RfcHJlZGljdGlvbnMkcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh0ZXN0X2RhdGEkaW50X3JhdGUgLSBtZWFuKHRlc3RfZGF0YSRpbnRfcmF0ZSkpXjIpKQoKIyBkaXNwbGF5IHRoZSBtZXRyaWNzCmNhdCgiVHJhaW5pbmcgTVNFOiIsIHJmLnRyYWluX21zZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1TRToiLCByZi50ZXN0X21zZSwgIlxuIikKY2F0KCJUcmFpbmluZyBSTVNFOiIsIHJmLnRyYWluX3Jtc2UsICJcbiIpCmNhdCgiVGVzdGluZyBSTVNFOiIsIHJmLnRlc3Rfcm1zZSwgIlxuIikKY2F0KCJUcmFpbmluZyBNQUU6IiwgcmYudHJhaW5fbWFlLCAiXG4iKQpjYXQoIlRlc3RpbmcgTUFFOiIsIHJmLnRlc3RfbWFlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFItc3F1YXJlZCAoUsKyKToiLCByZi50cmFpbl9yMiwgIlxuIikKY2F0KCJUZXN0aW5nIFItc3F1YXJlZCAoUsKyKToiLCByZi50ZXN0X3IyLCAiXG4iKQpgYGAKQm9vc3RpbmcKYGBge3J9CiMjIyMgQm9vc3RpbmcgIyMjIwoKIyBkZWZpbmUgdGhlIHRhcmdldCB2YXJpYWJsZSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKeGdiLnlfdHJhaW4gPC0gdHJhaW5fZGF0YSRpbnRfcmF0ZQp4Z2IueV90ZXN0IDwtIHRlc3RfZGF0YSRpbnRfcmF0ZQoKIyBkZWZpbmUgdGhlIGZlYXR1cmUgbWF0cml4IGZvciB0cmFpbmluZyBhbmQgdGVzdGluZyAoZXhjbHVkZSB0aGUgdGFyZ2V0IHZhcmlhYmxlKQp4Z2IuWF90cmFpbiA8LSB0cmFpbl9kYXRhWywgLXdoaWNoKG5hbWVzKHRyYWluX2RhdGEpID09ICdpbnRfcmF0ZScpXQp4Z2IuWF90ZXN0IDwtIHRlc3RfZGF0YVssIC13aGljaChuYW1lcyh0ZXN0X2RhdGEpID09ICdpbnRfcmF0ZScpXQoKIyBmaXQgYSBncmFkaWVudCBib29zdGluZyByZWdyZXNzaW9uIG1vZGVsIHVzaW5nIHhnYm9vc3QKeGdiIDwtIHhnYm9vc3QoCiAgZGF0YSA9IGFzLm1hdHJpeCh4Z2IuWF90cmFpbiksCiAgbGFiZWwgPSB4Z2IueV90cmFpbiwKICBucm91bmRzID0gMTAwLAogIHZlcmJvc2UgPSAwCikKCiMgbWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdHJhaW5pbmcgYW5kIHRlc3RpbmcgZGF0YQp4Z2IudHJhaW5fcHJlZGljdGlvbnMgPC0gcHJlZGljdCh4Z2IsIG5ld2RhdGEgPSBhcy5tYXRyaXgoeGdiLlhfdHJhaW4pKQp4Z2IudGVzdF9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KHhnYiwgbmV3ZGF0YSA9IGFzLm1hdHJpeCh4Z2IuWF90ZXN0KSkKCiMgY2FsY3VsYXRlIE1lYW4gU3F1YXJlZCBFcnJvciAoTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKeGdiLnRyYWluX21zZSA8LSBtZWFuKCh4Z2IudHJhaW5fcHJlZGljdGlvbnMgLSB4Z2IueV90cmFpbileMikKeGdiLnRlc3RfbXNlIDwtIG1lYW4oKHhnYi50ZXN0X3ByZWRpY3Rpb25zIC0geGdiLnlfdGVzdCleMikKCiMgY2FsY3VsYXRlIFJvb3QgTWVhbiBTcXVhcmVkIEVycm9yIChSTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKeGdiLnRyYWluX3Jtc2UgPC0gc3FydCh4Z2IudHJhaW5fbXNlKQp4Z2IudGVzdF9ybXNlIDwtIHNxcnQoeGdiLnRlc3RfbXNlKQoKIyBjYWxjdWxhdGUgTWVhbiBBYnNvbHV0ZSBFcnJvciAoTUFFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKeGdiLnRyYWluX21hZSA8LSBtZWFuKGFicyh4Z2IudHJhaW5fcHJlZGljdGlvbnMgLSB4Z2IueV90cmFpbikpCnhnYi50ZXN0X21hZSA8LSBtZWFuKGFicyh4Z2IudGVzdF9wcmVkaWN0aW9ucyAtIHhnYi55X3Rlc3QpKQoKIyBjYWxjdWxhdGUgUi1zcXVhcmVkIChSwrIpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwp4Z2IudHJhaW5fcjIgPC0gMSAtIChzdW0oKHhnYi55X3RyYWluIC0geGdiLnRyYWluX3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgoeGdiLnlfdHJhaW4gLSBtZWFuKHhnYi55X3RyYWluKSleMikpCnhnYi50ZXN0X3IyIDwtIDEgLSAoc3VtKCh4Z2IueV90ZXN0IC0geGdiLnRlc3RfcHJlZGljdGlvbnMpXjIpIC8gc3VtKCh4Z2IueV90ZXN0IC0gbWVhbih4Z2IueV90ZXN0KSleMikpCgojIGRpc3BsYXkgdGhlIG1ldHJpY3MKY2F0KCJUcmFpbmluZyBNU0U6IiwgeGdiLnRyYWluX21zZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1TRToiLCB4Z2IudGVzdF9tc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgUk1TRToiLCB4Z2IudHJhaW5fcm1zZSwgIlxuIikKY2F0KCJUZXN0aW5nIFJNU0U6IiwgeGdiLnRlc3Rfcm1zZSwgIlxuIikKY2F0KCJUcmFpbmluZyBNQUU6IiwgeGdiLnRyYWluX21hZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1BRToiLCB4Z2IudGVzdF9tYWUsICJcbiIpCmNhdCgiVHJhaW5pbmcgUi1zcXVhcmVkIChSwrIpOiIsIHhnYi50cmFpbl9yMiwgIlxuIikKY2F0KCJUZXN0aW5nIFItc3F1YXJlZCAoUsKyKToiLCB4Z2IudGVzdF9yMiwgIlxuIikKYGBgCkZvbGxvd2luZywgYSBzY2F0dGVyIHBsb3Qgb2YgYWN0dWFsIHZzIHByZWRpY3RlZCB0cmFpbmluZyB2YWx1ZXMgZm9yIGVhY2ggbW9kZWwgaXMgcGxvdC4KVGhpcyBwbG90IGhlbHBzIHVzIHZpc3VhbGl6ZSBob3cgd2VsbCBlYWNoIG1vZGVsJ3MgcHJlZGljdGlvbnMgYWxpZ24gd2l0aCB0aGUgYWN0dWFsIGRhdGEgcG9pbnRzLgpgYGB7cn0KIyBjcmVhdGUgYSBzY2F0dGVyIHBsb3QgZnVuY3Rpb24KY3JlYXRlX3NjYXR0ZXJfcGxvdCA8LSBmdW5jdGlvbihhY3R1YWxfdmFsdWVzLCBwcmVkaWN0ZWRfdmFsdWVzLCBtb2RlbF9uYW1lKSB7CiAgbW9kZWxfY29tcGFyaXNvbl9kYXRhIDwtIGRhdGEuZnJhbWUoCiAgICBBY3R1YWwgPSBhY3R1YWxfdmFsdWVzLAogICAgUHJlZGljdGVkID0gcHJlZGljdGVkX3ZhbHVlcwogICkKICAKICBzY2F0dGVyX3Bsb3QgPC0gZ2dwbG90KG1vZGVsX2NvbXBhcmlzb25fZGF0YSwgYWVzKHggPSBBY3R1YWwsIHkgPSBQcmVkaWN0ZWQpKSArCiAgICBnZW9tX3BvaW50KCkgKwogICAgZ2VvbV9hYmxpbmUoaW50ZXJjZXB0ID0gMCwgc2xvcGUgPSAxLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJyZWQiKSArICAjIGFkZCBhIGRpYWdvbmFsIHJlZmVyZW5jZSBsaW5lCiAgICBsYWJzKHggPSAiQWN0dWFsIFRyYWluaW5nIFZhbHVlcyIsIHkgPSAiUHJlZGljdGVkIFRyYWluaW5nIFZhbHVlcyIsIHRpdGxlID0gbW9kZWxfbmFtZSkgKwogICAgdGhlbWVfbWluaW1hbCgpICsKICAgIHlsaW0oLTUwLCA1MCkKICAKICByZXR1cm4oc2NhdHRlcl9wbG90KQp9CgojIGNyZWF0ZSBzY2F0dGVyIHBsb3RzIGZvciBlYWNoIG1vZGVsCmxtX3NjYXR0ZXJfcGxvdCA8LSBjcmVhdGVfc2NhdHRlcl9wbG90KAogIGFjdHVhbF92YWx1ZXMgPSB0cmFpbl9kYXRhJGludF9yYXRlLAogIHByZWRpY3RlZF92YWx1ZXMgPSBsbS50cmFpbl9wcmVkaWN0aW9ucywKICBtb2RlbF9uYW1lID0gIkxpbmVhciBSZWdyZXNzaW9uIgopCgpyZl9zY2F0dGVyX3Bsb3QgPC0gY3JlYXRlX3NjYXR0ZXJfcGxvdCgKICBhY3R1YWxfdmFsdWVzID0gdHJhaW5fZGF0YSRpbnRfcmF0ZSwKICBwcmVkaWN0ZWRfdmFsdWVzID0gcmYudHJhaW5fcHJlZGljdGlvbnMkcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJSYW5kb20gRm9yZXN0IgopCgp4Z2Jfc2NhdHRlcl9wbG90IDwtIGNyZWF0ZV9zY2F0dGVyX3Bsb3QoCiAgYWN0dWFsX3ZhbHVlcyA9IHhnYi55X3RyYWluLAogIHByZWRpY3RlZF92YWx1ZXMgPSB4Z2IudHJhaW5fcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJYR0Jvb3N0IgopCgojIGRpc3BsYXkgdGhlIHNjYXR0ZXIgcGxvdHMgc2VwYXJhdGVseQpwcmludChsbV9zY2F0dGVyX3Bsb3QpCnByaW50KHJmX3NjYXR0ZXJfcGxvdCkKcHJpbnQoeGdiX3NjYXR0ZXJfcGxvdCkKYGBgCkZvbGxvd2luZywgYSBzY2F0dGVyIHBsb3Qgb2YgYWN0dWFsIHZzIHByZWRpY3RlZCB0ZXN0aW5nIHZhbHVlcyBmb3IgZWFjaCBtb2RlbCBpcyBwbG90LgpUaGlzIHBsb3QgaGVscHMgdXMgdmlzdWFsaXplIGhvdyB3ZWxsIGVhY2ggbW9kZWwncyBwcmVkaWN0aW9ucyBhbGlnbiB3aXRoIHRoZSBhY3R1YWwgZGF0YSBwb2ludHMuCmBgYHtyfQojIGNyZWF0ZSBhIHNjYXR0ZXIgcGxvdCBmdW5jdGlvbgpjcmVhdGVfc2NhdHRlcl9wbG90IDwtIGZ1bmN0aW9uKGFjdHVhbF92YWx1ZXMsIHByZWRpY3RlZF92YWx1ZXMsIG1vZGVsX25hbWUpIHsKICBtb2RlbF9jb21wYXJpc29uX2RhdGEgPC0gZGF0YS5mcmFtZSgKICAgIEFjdHVhbCA9IGFjdHVhbF92YWx1ZXMsCiAgICBQcmVkaWN0ZWQgPSBwcmVkaWN0ZWRfdmFsdWVzCiAgKQogIAogIHNjYXR0ZXJfcGxvdCA8LSBnZ3Bsb3QobW9kZWxfY29tcGFyaXNvbl9kYXRhLCBhZXMoeCA9IEFjdHVhbCwgeSA9IFByZWRpY3RlZCkpICsKICAgIGdlb21fcG9pbnQoKSArCiAgICBnZW9tX2FibGluZShpbnRlcmNlcHQgPSAwLCBzbG9wZSA9IDEsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yID0gInJlZCIpICsgICMgYWRkIGEgZGlhZ29uYWwgcmVmZXJlbmNlIGxpbmUKICAgIGxhYnMoeCA9ICJBY3R1YWwgVGVzdGluZyBWYWx1ZXMiLCB5ID0gIlByZWRpY3RlZCBUZXN0aW5nIFZhbHVlcyIsIHRpdGxlID0gbW9kZWxfbmFtZSkgKwogICAgdGhlbWVfbWluaW1hbCgpICsKICAgIHlsaW0oLTUwLCA1MCkgKwogICAgeGxpbSgwLCA0MCkKICAKICByZXR1cm4oc2NhdHRlcl9wbG90KQp9CgojIGNyZWF0ZSBzY2F0dGVyIHBsb3RzIGZvciBlYWNoIG1vZGVsCmxtX3NjYXR0ZXJfcGxvdCA8LSBjcmVhdGVfc2NhdHRlcl9wbG90KAogIGFjdHVhbF92YWx1ZXMgPSB0ZXN0X2RhdGEkaW50X3JhdGUsCiAgcHJlZGljdGVkX3ZhbHVlcyA9IGxtLnRlc3RfcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJMaW5lYXIgUmVncmVzc2lvbiIKKQoKcmZfc2NhdHRlcl9wbG90IDwtIGNyZWF0ZV9zY2F0dGVyX3Bsb3QoCiAgYWN0dWFsX3ZhbHVlcyA9IHRlc3RfZGF0YSRpbnRfcmF0ZSwKICBwcmVkaWN0ZWRfdmFsdWVzID0gcmYudGVzdF9wcmVkaWN0aW9ucyRwcmVkaWN0aW9ucywKICBtb2RlbF9uYW1lID0gIlJhbmRvbSBGb3Jlc3QiCikKCnhnYl9zY2F0dGVyX3Bsb3QgPC0gY3JlYXRlX3NjYXR0ZXJfcGxvdCgKICBhY3R1YWxfdmFsdWVzID0geGdiLnlfdGVzdCwKICBwcmVkaWN0ZWRfdmFsdWVzID0geGdiLnRlc3RfcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJYR0Jvb3N0IgopCgojIGRpc3BsYXkgdGhlIHNjYXR0ZXIgcGxvdHMgc2VwYXJhdGVseQpwcmludChsbV9zY2F0dGVyX3Bsb3QpCnByaW50KHJmX3NjYXR0ZXJfcGxvdCkKcHJpbnQoeGdiX3NjYXR0ZXJfcGxvdCkKYGBgCgpSZXNpZHVhbCBwbG90cyBjYW4gaGVscCBpZGVudGlmeSBwYXR0ZXJucyBpbiBwcmVkaWN0aW9uIGVycm9ycyBhbmQgYXNzZXNzIHdoZXRoZXIgdGhlIGFzc3VtcHRpb25zIG9mIGxpbmVhciByZWdyZXNzaW9uIChpZiBhcHBsaWNhYmxlKSBhcmUgbWV0LgpgYGB7cn0KIyBjcmVhdGUgYSByZXNpZHVhbCBwbG90IGZ1bmN0aW9uCmNyZWF0ZV9yZXNpZHVhbF9wbG90IDwtIGZ1bmN0aW9uKGFjdHVhbF92YWx1ZXMsIHByZWRpY3RlZF92YWx1ZXMsIG1vZGVsX25hbWUpIHsKICByZXNpZHVhbHMgPC0gYWN0dWFsX3ZhbHVlcyAtIHByZWRpY3RlZF92YWx1ZXMKICByZXNpZHVhbF9kYXRhIDwtIGRhdGEuZnJhbWUoCiAgICBQcmVkaWN0ZWQgPSBwcmVkaWN0ZWRfdmFsdWVzLAogICAgUmVzaWR1YWxzID0gcmVzaWR1YWxzCiAgKQogIAogIHJlc2lkdWFsX3Bsb3QgPC0gZ2dwbG90KHJlc2lkdWFsX2RhdGEsIGFlcyh4ID0gUHJlZGljdGVkLCB5ID0gUmVzaWR1YWxzKSkgKwogICAgZ2VvbV9wb2ludCgpICsKICAgIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yID0gInJlZCIpICsgICMgUmVkIGhvcml6b250YWwgcmVmZXJlbmNlIGxpbmUKICAgIGxhYnMoeCA9ICJQcmVkaWN0ZWQgVmFsdWVzIiwgeSA9ICJSZXNpZHVhbHMiLCB0aXRsZSA9IHBhc3RlKCJSZXNpZHVhbCBQbG90IC0iLCBtb2RlbF9uYW1lKSkgKwogICAgdGhlbWVfbWluaW1hbCgpICsKICAgIHlsaW0oLTMwLCAzMCkgKwogICAgeGxpbSgwLCA0MCkKICAKICByZXR1cm4ocmVzaWR1YWxfcGxvdCkKfQoKIyBjcmVhdGUgcmVzaWR1YWwgcGxvdHMgZm9yIGVhY2ggbW9kZWwKbG1fcmVzaWR1YWxfcGxvdCA8LSBjcmVhdGVfcmVzaWR1YWxfcGxvdCgKICBhY3R1YWxfdmFsdWVzID0gdHJhaW5fZGF0YSRpbnRfcmF0ZSwKICBwcmVkaWN0ZWRfdmFsdWVzID0gbG0udHJhaW5fcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJMaW5lYXIgUmVncmVzc2lvbiIKKQoKcmZfcmVzaWR1YWxfcGxvdCA8LSBjcmVhdGVfcmVzaWR1YWxfcGxvdCgKICBhY3R1YWxfdmFsdWVzID0gdHJhaW5fZGF0YSRpbnRfcmF0ZSwKICBwcmVkaWN0ZWRfdmFsdWVzID0gcmYudHJhaW5fcHJlZGljdGlvbnMkcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJSYW5kb20gRm9yZXN0IgopCgp4Z2JfcmVzaWR1YWxfcGxvdCA8LSBjcmVhdGVfcmVzaWR1YWxfcGxvdCgKICBhY3R1YWxfdmFsdWVzID0geGdiLnlfdHJhaW4sCiAgcHJlZGljdGVkX3ZhbHVlcyA9IHhnYi50cmFpbl9wcmVkaWN0aW9ucywKICBtb2RlbF9uYW1lID0gIlhHQm9vc3QiCikKCiMgZGlzcGxheSB0aGUgcmVzaWR1YWwgcGxvdHMgc2VwYXJhdGVseQpwcmludChsbV9yZXNpZHVhbF9wbG90KQpwcmludChyZl9yZXNpZHVhbF9wbG90KQpwcmludCh4Z2JfcmVzaWR1YWxfcGxvdCkKYGBgCgpgYGB7cn0KIyBjcmVhdGUgYSBkZW5zaXR5IHBsb3QgZnVuY3Rpb24gZm9yIHJlc2lkdWFscwpjcmVhdGVfcmVzaWR1YWxfZGVuc2l0eV9wbG90IDwtIGZ1bmN0aW9uKGFjdHVhbF92YWx1ZXMsIHByZWRpY3RlZF92YWx1ZXMsIG1vZGVsX25hbWUpIHsKICByZXNpZHVhbHMgPC0gYWN0dWFsX3ZhbHVlcyAtIHByZWRpY3RlZF92YWx1ZXMKICByZXNpZHVhbF9kYXRhIDwtIGRhdGEuZnJhbWUoUmVzaWR1YWxzID0gcmVzaWR1YWxzKQogIAogIGRlbnNpdHlfcGxvdCA8LSBnZ3Bsb3QocmVzaWR1YWxfZGF0YSwgYWVzKHggPSBSZXNpZHVhbHMpKSArCiAgICBnZW9tX2RlbnNpdHkoZmlsbCA9ICJza3libHVlIiwgY29sb3IgPSAiYmxhY2siLCBhbHBoYSA9IDAuNykgKwogICAgbGFicyh4ID0gIlJlc2lkdWFscyIsIHkgPSAiRGVuc2l0eSIsIHRpdGxlID0gcGFzdGUoIlJlc2lkdWFsIERlbnNpdHkgUGxvdCAtIiwgbW9kZWxfbmFtZSkpICsKICAgIHRoZW1lX21pbmltYWwoKSArCiAgICB4bGltKC0zMCwzMCkgKyAKICAgIHlsaW0oMCwgMC4zNSkKICAgIAogIAogIHJldHVybihkZW5zaXR5X3Bsb3QpCn0KCiMgY3JlYXRlIGRlbnNpdHkgcGxvdHMgZm9yIHJlc2lkdWFscyBmb3IgZWFjaCBtb2RlbApsbV9yZXNpZHVhbF9kZW5zaXR5X3Bsb3QgPC0gY3JlYXRlX3Jlc2lkdWFsX2RlbnNpdHlfcGxvdCgKICBhY3R1YWxfdmFsdWVzID0gdHJhaW5fZGF0YSRpbnRfcmF0ZSwKICBwcmVkaWN0ZWRfdmFsdWVzID0gbG0udHJhaW5fcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJMaW5lYXIgUmVncmVzc2lvbiIKKQoKcmZfcmVzaWR1YWxfZGVuc2l0eV9wbG90IDwtIGNyZWF0ZV9yZXNpZHVhbF9kZW5zaXR5X3Bsb3QoCiAgYWN0dWFsX3ZhbHVlcyA9IHRyYWluX2RhdGEkaW50X3JhdGUsCiAgcHJlZGljdGVkX3ZhbHVlcyA9IHJmLnRyYWluX3ByZWRpY3Rpb25zJHByZWRpY3Rpb25zLAogIG1vZGVsX25hbWUgPSAiUmFuZG9tIEZvcmVzdCIKKQoKeGdiX3Jlc2lkdWFsX2RlbnNpdHlfcGxvdCA8LSBjcmVhdGVfcmVzaWR1YWxfZGVuc2l0eV9wbG90KAogIGFjdHVhbF92YWx1ZXMgPSB4Z2IueV90cmFpbiwKICBwcmVkaWN0ZWRfdmFsdWVzID0geGdiLnRyYWluX3ByZWRpY3Rpb25zLAogIG1vZGVsX25hbWUgPSAiWEdCb29zdCIKKQoKIyBkaXNwbGF5IHRoZSBkZW5zaXR5IHBsb3RzIHNlcGFyYXRlbHkKcHJpbnQobG1fcmVzaWR1YWxfZGVuc2l0eV9wbG90KQpwcmludChyZl9yZXNpZHVhbF9kZW5zaXR5X3Bsb3QpCnByaW50KHhnYl9yZXNpZHVhbF9kZW5zaXR5X3Bsb3QpCmBgYAoKVGhpcyB2aXN1YWxpemF0aW9uIGNhbiBoZWxwIHlvdSBjb21wYXJlIHRoZSBkaXN0cmlidXRpb24gb2YgcHJlZGljdGlvbiBlcnJvcnMgYWNyb3NzIG1vZGVscyB0aHJvdWdoIGhpc3RvZ3JhbXMuCgpgYGB7cn0KIyBjcmVhdGUgYSBoaXN0b2dyYW0gcGxvdCBmdW5jdGlvbiBmb3IgcmVzaWR1YWxzIHdpdGggYSByZWQgZGVuc2l0eSBjdXJ2ZQpjcmVhdGVfcmVzaWR1YWxfaGlzdG9ncmFtX3Bsb3QgPC0gZnVuY3Rpb24oYWN0dWFsX3ZhbHVlcywgcHJlZGljdGVkX3ZhbHVlcywgbW9kZWxfbmFtZSkgewogIHJlc2lkdWFscyA8LSBhY3R1YWxfdmFsdWVzIC0gcHJlZGljdGVkX3ZhbHVlcwogIHJlc2lkdWFsX2RhdGEgPC0gZGF0YS5mcmFtZShSZXNpZHVhbHMgPSByZXNpZHVhbHMpCiAgCiAgaGlzdG9ncmFtX3Bsb3QgPC0gZ2dwbG90KHJlc2lkdWFsX2RhdGEsIGFlcyh4ID0gUmVzaWR1YWxzKSkgKwogICAgZ2VvbV9oaXN0b2dyYW0oYWVzKHkgPSBhZnRlcl9zdGF0KGRlbnNpdHkpKSwgYmlucyA9IDMwLCBmaWxsID0gInNreWJsdWUiLCBjb2xvciA9ICJibGFjayIsIGFscGhhID0gMC43KSArICAjIHVzZSBkZW5zaXR5IG9uIHRoZSB5LWF4aXMgZm9yIHRoZSBoaXN0b2dyYW0KICAgIGdlb21fZGVuc2l0eShjb2xvciA9ICJyZWQiLCBsaW5ld2lkdGggPSAxLjUpICsgICMgYWRkIHRoZSBkZW5zaXR5IHBsb3QgaW4gcmVkCiAgICBsYWJzKHggPSAiUmVzaWR1YWxzIiwgeSA9ICJEZW5zaXR5IiwgdGl0bGUgPSBwYXN0ZSgiUmVzaWR1YWwgSGlzdG9ncmFtIFBsb3Qgd2l0aCBEZW5zaXR5IEN1cnZlIC0iLCBtb2RlbF9uYW1lKSkgKwogICAgdGhlbWVfbWluaW1hbCgpICsKICAgIHhsaW0oLTIwLDIwKSArIAogICAgeWxpbSgwLCAwLjMpCiAgCiAgcmV0dXJuKGhpc3RvZ3JhbV9wbG90KQp9CgojIGNyZWF0ZSBoaXN0b2dyYW0gcGxvdHMgZm9yIHJlc2lkdWFscyBmb3IgZWFjaCBtb2RlbApsbV9yZXNpZHVhbF9oaXN0b2dyYW1fcGxvdCA8LSBjcmVhdGVfcmVzaWR1YWxfaGlzdG9ncmFtX3Bsb3QoCiAgYWN0dWFsX3ZhbHVlcyA9IHRyYWluX2RhdGEkaW50X3JhdGUsCiAgcHJlZGljdGVkX3ZhbHVlcyA9IGxtLnRyYWluX3ByZWRpY3Rpb25zLAogIG1vZGVsX25hbWUgPSAiTGluZWFyIFJlZ3Jlc3Npb24iCikKCnJmX3Jlc2lkdWFsX2hpc3RvZ3JhbV9wbG90IDwtIGNyZWF0ZV9yZXNpZHVhbF9oaXN0b2dyYW1fcGxvdCgKICBhY3R1YWxfdmFsdWVzID0gdHJhaW5fZGF0YSRpbnRfcmF0ZSwKICBwcmVkaWN0ZWRfdmFsdWVzID0gcmYudHJhaW5fcHJlZGljdGlvbnMkcHJlZGljdGlvbnMsCiAgbW9kZWxfbmFtZSA9ICJSYW5kb20gRm9yZXN0IgopCgp4Z2JfcmVzaWR1YWxfaGlzdG9ncmFtX3Bsb3QgPC0gY3JlYXRlX3Jlc2lkdWFsX2hpc3RvZ3JhbV9wbG90KAogIGFjdHVhbF92YWx1ZXMgPSB4Z2IueV90cmFpbiwKICBwcmVkaWN0ZWRfdmFsdWVzID0geGdiLnRyYWluX3ByZWRpY3Rpb25zLAogIG1vZGVsX25hbWUgPSAiWEdCb29zdCIKKQoKIyBkaXNwbGF5IHRoZSBoaXN0b2dyYW0gcGxvdHMgc2VwYXJhdGVseQpwcmludChsbV9yZXNpZHVhbF9oaXN0b2dyYW1fcGxvdCkKcHJpbnQocmZfcmVzaWR1YWxfaGlzdG9ncmFtX3Bsb3QpCnByaW50KHhnYl9yZXNpZHVhbF9oaXN0b2dyYW1fcGxvdCkKYGBgCgpGb3IgZWFjaCBtb2RlbCBhIGJhciBjaGFydCB0aGF0IGRpc3BsYXlzIHRoZSBSLXNxdWFyZWQgKGNvZWZmaWNpZW50IG9mIGRldGVybWluYXRpb24pIHZhbHVlcyBpcyBjcmVhdGVkLgpSLXNxdWFyZWQgbWVhc3VyZXMgdGhlIHByb3BvcnRpb24gb2YgdmFyaWFuY2UgaW4gdGhlIHRhcmdldCB2YXJpYWJsZSBleHBsYWluZWQgYnkgdGhlIG1vZGVsLiBIaWdoZXIgUi1zcXVhcmVkIHZhbHVlcyBpbmRpY2F0ZSBiZXR0ZXIgbW9kZWwgZml0LgpgYGB7cn0KIyBjcmVhdGUgYSBkYXRhIGZyYW1lIHdpdGggUi1zcXVhcmVkIHZhbHVlcyBmb3IgZWFjaCBtb2RlbAptb2RlbF9uYW1lcyA8LSBjKCJMaW5lYXIgUmVncmVzc2lvbiIsICJSYW5kb20gRm9yZXN0IiwgIlhHQm9vc3QiKQpyX3NxdWFyZWRfdmFsdWVzX3RyYWluIDwtIGMoCiAgbG0udHJhaW5fcjIsCiAgcmYudHJhaW5fcjIsCiAgeGdiLnRyYWluX3IyCikKcl9zcXVhcmVkX3ZhbHVlc190ZXN0IDwtIGMoCiAgbG0udGVzdF9yMiwKICByZi50ZXN0X3IyLAogIHhnYi50ZXN0X3IyCikKCnJfc3F1YXJlZF9kYXRhX3RyYWluIDwtIGRhdGEuZnJhbWUoTW9kZWwgPSBmYWN0b3IobW9kZWxfbmFtZXMpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBSX3NxdWFyZWQgPSByX3NxdWFyZWRfdmFsdWVzX3RyYWluKQpyX3NxdWFyZWRfZGF0YV90ZXN0IDwtIGRhdGEuZnJhbWUoTW9kZWwgPSBmYWN0b3IobW9kZWxfbmFtZXMpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBSX3NxdWFyZWQgPSByX3NxdWFyZWRfdmFsdWVzX3Rlc3QpCgojIGNyZWF0ZSB0aGUgUi1zcXVhcmVkIGNvbXBhcmlzb24gYmFyIGNoYXJ0CnJfc3F1YXJlZF9iYXJfY2hhcnRfdHJhaW4gPC0gZ2dwbG90KHJfc3F1YXJlZF9kYXRhX3RyYWluLCBhZXMoeCA9IE1vZGVsLCB5ID0gUl9zcXVhcmVkLCBmaWxsID0gTW9kZWwpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICJpZGVudGl0eSIpICsKICBsYWJzKHggPSAiTW9kZWwiLCB5ID0gIlItc3F1YXJlZCAoUsKyKSIsIHRpdGxlID0gIlItc3F1YXJlZCBDb21wYXJpc29uIFRyYWluaW5nIikgKwogIHRoZW1lX21pbmltYWwoKSArCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgaGp1c3QgPSAxKSkgKyAKICB5bGltKDAsMSkKcl9zcXVhcmVkX2Jhcl9jaGFydF90ZXN0IDwtIGdncGxvdChyX3NxdWFyZWRfZGF0YV90ZXN0LCBhZXMoeCA9IE1vZGVsLCB5ID0gUl9zcXVhcmVkLCBmaWxsID0gTW9kZWwpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICJpZGVudGl0eSIpICsKICBsYWJzKHggPSAiTW9kZWwiLCB5ID0gIlItc3F1YXJlZCAoUsKyKSIsIHRpdGxlID0gIlItc3F1YXJlZCBDb21wYXJpc29uIFRlc3RpbmciKSArCiAgdGhlbWVfbWluaW1hbCgpICsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCBoanVzdCA9IDEpKSArIAogIHlsaW0oMCwxKQoKIyBkaXNwbGF5IHRoZSBSLXNxdWFyZWQgY29tcGFyaXNvbiBiYXIgY2hhcnQKcHJpbnQocl9zcXVhcmVkX2Jhcl9jaGFydF90cmFpbikKcHJpbnQocl9zcXVhcmVkX2Jhcl9jaGFydF90ZXN0KQpgYGAKQSBiYXIgY2hhcnQgdGhhdCBjb21wYXJlcyB0aGUgTUFFIG9yIFJNU0UgdmFsdWVzLCBpcyBnZW5lcmF0ZWQgZm9yIGVhY2ggbW9kZWwuClRoZXNlIG1ldHJpY3MgcXVhbnRpZnkgdGhlIGF2ZXJhZ2UgcHJlZGljdGlvbiBlcnJvcnMgb2YgZWFjaCBtb2RlbCwgYW5kIGxvd2VyIHZhbHVlcyBhcmUgcHJlZmVycmVkLgpgYGB7cn0KIyBjcmVhdGUgYSBkYXRhIGZyYW1lIHdpdGggTUFFIGFuZCBSTVNFIHZhbHVlcyBmb3IgZWFjaCBtb2RlbAptb2RlbF9uYW1lcyA8LSBjKCJMaW5lYXIgUmVncmVzc2lvbiIsICJSYW5kb20gRm9yZXN0IiwgIlhHQm9vc3QiLCJMaW5lYXIgUmVncmVzc2lvbiIsICJSYW5kb20gRm9yZXN0IiwgIlhHQm9vc3QiKQplcnJvcl92YWx1ZXNfdHJhaW4gPC0gYygKICBsbS50cmFpbl9tYWUsCiAgcmYudHJhaW5fbWFlLAogIHhnYi50cmFpbl9tYWUsCiAgbG0udHJhaW5fcm1zZSwKICByZi50cmFpbl9ybXNlLAogIHhnYi50cmFpbl9ybXNlCikKZXJyb3JfdmFsdWVzX3Rlc3QgPC0gYygKICBsbS50ZXN0X21hZSwKICByZi50ZXN0X21hZSwKICB4Z2IudGVzdF9tYWUsCiAgbG0udGVzdF9ybXNlLAogIHJmLnRlc3Rfcm1zZSwKICB4Z2IudGVzdF9ybXNlCikKZXJyb3JfdHlwZSA8LSBjKAogICJNQUUiLCAiTUFFIiwgIk1BRSIsIlJNU0UiLCJSTVNFIiwiUk1TRSIKKQptb2RlbF9lcnJvcnNfdHJhaW4gPC0gZGF0YS5mcmFtZShNb2RlbCA9IGZhY3Rvcihtb2RlbF9uYW1lcywgbGV2ZWxzID0gYygiTGluZWFyIFJlZ3Jlc3Npb24iLCAiUmFuZG9tIEZvcmVzdCIsICJYR0Jvb3N0IikpLAogICAgICAgICAgICAgICAgRXJyb3IgPSBlcnJvcl92YWx1ZXNfdHJhaW4sIFR5cGUgPSBlcnJvcl90eXBlKQptb2RlbF9lcnJvcnNfdGVzdCA8LSBkYXRhLmZyYW1lKE1vZGVsID0gZmFjdG9yKG1vZGVsX25hbWVzLCBsZXZlbHMgPSBjKCJMaW5lYXIgUmVncmVzc2lvbiIsICJSYW5kb20gRm9yZXN0IiwgIlhHQm9vc3QiKSksCiAgICAgICAgICAgICAgICBFcnJvciA9IGVycm9yX3ZhbHVlc190ZXN0LCBUeXBlID0gZXJyb3JfdHlwZSkKIyBjcmVhdGUgdGhlIE1BRSBvciBSTVNFIGNvbXBhcmlzb24gYmFyIGNoYXJ0CmVycm9yX2Jhcl9jaGFydF90cmFpbiA8LSBnZ3Bsb3QobW9kZWxfZXJyb3JzX3RyYWluLCBhZXMoeCA9IE1vZGVsLCB5ID0gRXJyb3IsIGZpbGwgPSBUeXBlKSkgKwogIGdlb21fYmFyKHN0YXQgPSAiaWRlbnRpdHkiLCBwb3NpdGlvbiA9ICJkb2RnZSIpICsKICBsYWJzKHggPSAiTW9kZWwiLCB5ID0gIkVycm9yIFZhbHVlIiwgdGl0bGUgPSAiVHJhaW5pbmcgTUFFIGFuZCBSTVNFIENvbXBhcmlzb24iKSArCiAgdGhlbWVfbWluaW1hbCgpICsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCBoanVzdCA9IDEpKSArIAogIHlsaW0oMCwgNCkKCmVycm9yX2Jhcl9jaGFydF90ZXN0IDwtIGdncGxvdChtb2RlbF9lcnJvcnNfdGVzdCwgYWVzKHggPSBNb2RlbCwgeSA9IEVycm9yLCBmaWxsID0gVHlwZSkpICsKICBnZW9tX2JhcihzdGF0ID0gImlkZW50aXR5IiwgcG9zaXRpb24gPSAiZG9kZ2UiKSArCiAgbGFicyh4ID0gIk1vZGVsIiwgeSA9ICJFcnJvciBWYWx1ZSIsIHRpdGxlID0gIlRlc3RpbmcgTUFFIGFuZCBSTVNFIENvbXBhcmlzb24iKSArCiAgdGhlbWVfbWluaW1hbCgpICsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCBoanVzdCA9IDEpKSArIAogIHlsaW0oMCwgNCkKCiMgZGlzcGxheSB0aGUgTUFFIGFuZCBSTVNFIGNvbXBhcmlzb24gYmFyIGNoYXJ0CnByaW50KGVycm9yX2Jhcl9jaGFydF90cmFpbikKcHJpbnQoZXJyb3JfYmFyX2NoYXJ0X3Rlc3QpCmBgYAoKCmBgYHtyfQojIyMjIFJhbmRvbSBGb3Jlc3QgRmVhdHVyZSBJbXBvcnRhbmNlIFBsb3QgIyMjIwp2MSA8LSB2aXAocmYsIHRpdGxlID0gIlJhbmdlciIsIG51bV9mZWF0dXJlcyA9IDIwKSAKcGxvdCh2MSkKYGBgCkZlYXR1cmUgU2VsZWN0aW9uIGZyb20gdGhlIHZhcmlhYmxlIGltcG9ydGFuY2UncyBhbmFseXNpczoKCmBgYHtyfQppbXAudmFyaWFibGVzIDwtIGxjX2RhdGFbLCB2MSRkYXRhJFZhcmlhYmxlXQppbXAudmFyaWFibGVzJGludF9yYXRlIDwtIGxjX2RhdGEkaW50X3JhdGUKaW1wLnRyYWluX2luZGljZXMgPC0gY3JlYXRlRGF0YVBhcnRpdGlvbihpbXAudmFyaWFibGVzJGludF9yYXRlLCBwID0gMC44LCBsaXN0ID0gRkFMU0UpCgojIGNyZWF0ZSB0cmFpbmluZyBhbmQgdGVzdGluZyBkYXRhc2V0cwppbXAudHJhaW5fZGF0YSA8LSBpbXAudmFyaWFibGVzW2ltcC50cmFpbl9pbmRpY2VzLCBdCmltcC50ZXN0X2RhdGEgPC0gaW1wLnZhcmlhYmxlc1staW1wLnRyYWluX2luZGljZXMsIF0KYGBgCgpgYGB7cn0KIyMjIyBMaW5lYXIgUmVncmVzc2lvbiB3aXRoIG9ubHkgaW1wb3J0YW5jZSB2YXJpYWJsZXMgIyMjIwoKaW1wLmxtLmZpdCA8LSBsbShpbnRfcmF0ZSB+IC4sIGRhdGEgPSBpbXAudHJhaW5fZGF0YSkKCiMgbWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdHJhaW5pbmcgYW5kIHRlc3RpbmcgZGF0YQppbXAubG0udHJhaW5fcHJlZGljdGlvbnMgPC0gcHJlZGljdChpbXAubG0uZml0LCBuZXdkYXRhID0gaW1wLnRyYWluX2RhdGEpCmltcC5sbS50ZXN0X3ByZWRpY3Rpb25zIDwtIHByZWRpY3QoaW1wLmxtLmZpdCwgbmV3ZGF0YSA9IGltcC50ZXN0X2RhdGEpCgojIGNhbGN1bGF0ZSBNZWFuIFNxdWFyZWQgRXJyb3IgKE1TRSkgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCmltcC5sbS50cmFpbl9tc2UgPC0gbWVhbigoaW1wLmxtLnRyYWluX3ByZWRpY3Rpb25zIC0gaW1wLnRyYWluX2RhdGEkaW50X3JhdGUpXjIpCmltcC5sbS50ZXN0X21zZSA8LSBtZWFuKChpbXAubG0udGVzdF9wcmVkaWN0aW9ucyAtIGltcC50ZXN0X2RhdGEkaW50X3JhdGUpXjIpCgojIGNhbGN1bGF0ZSBSb290IE1lYW4gU3F1YXJlZCBFcnJvciAoUk1TRSkgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCmltcC5sbS50cmFpbl9ybXNlIDwtIHNxcnQoaW1wLmxtLnRyYWluX21zZSkKaW1wLmxtLnRlc3Rfcm1zZSA8LSBzcXJ0KGltcC5sbS50ZXN0X21zZSkKCiMgY2FsY3VsYXRlIE1lYW4gQWJzb2x1dGUgRXJyb3IgKE1BRSkgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCmltcC5sbS50cmFpbl9tYWUgPC0gbWVhbihhYnMoaW1wLmxtLnRyYWluX3ByZWRpY3Rpb25zIC0gaW1wLnRyYWluX2RhdGEkaW50X3JhdGUpKQppbXAubG0udGVzdF9tYWUgPC0gbWVhbihhYnMoaW1wLmxtLnRlc3RfcHJlZGljdGlvbnMgLSBpbXAudGVzdF9kYXRhJGludF9yYXRlKSkKCiMgY2FsY3VsYXRlIFItc3F1YXJlZCAoUsKyKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKaW1wLmxtLnRyYWluX3IyIDwtIDEgLSAoc3VtKChpbXAudHJhaW5fZGF0YSRpbnRfcmF0ZSAtIGltcC5sbS50cmFpbl9wcmVkaWN0aW9ucyleMikgLyBzdW0oKGltcC50cmFpbl9kYXRhJGludF9yYXRlIC0gbWVhbihpbXAudHJhaW5fZGF0YSRpbnRfcmF0ZSkpXjIpKQppbXAubG0udGVzdF9yMiA8LSAxIC0gKHN1bSgoaW1wLnRlc3RfZGF0YSRpbnRfcmF0ZSAtIGltcC5sbS50ZXN0X3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgoaW1wLnRlc3RfZGF0YSRpbnRfcmF0ZSAtIG1lYW4oaW1wLnRlc3RfZGF0YSRpbnRfcmF0ZSkpXjIpKQoKIyBkaXNwbGF5IHRoZSBtZXRyaWNzCmNhdCgiVHJhaW5pbmcgTVNFOiIsIGltcC5sbS50cmFpbl9tc2UsICJcbiIpCmNhdCgiVGVzdGluZyBNU0U6IiwgaW1wLmxtLnRlc3RfbXNlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFJNU0U6IiwgaW1wLmxtLnRyYWluX3Jtc2UsICJcbiIpCmNhdCgiVGVzdGluZyBSTVNFOiIsIGltcC5sbS50ZXN0X3Jtc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgTUFFOiIsIGltcC5sbS50cmFpbl9tYWUsICJcbiIpCmNhdCgiVGVzdGluZyBNQUU6IiwgaW1wLmxtLnRlc3RfbWFlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFItc3F1YXJlZCAoUsKyKToiLCBpbXAubG0udHJhaW5fcjIsICJcbiIpCmNhdCgiVGVzdGluZyBSLXNxdWFyZWQgKFLCsik6IiwgaW1wLmxtLnRlc3RfcjIsICJcbiIpCmBgYAoKYGBge3J9CiMjIyMgUmFuZG9tIEZvcmVzdCB3aXRoIG9ubHkgaW1wb3J0YW5jZSB2YXJpYWJsZXMgIyMjIwoKIyB0cmFpbiBhIFJhbmRvbSBGb3Jlc3QgbW9kZWwKaW1wLnJmIDwtIHJhbmdlcihmb3JtdWxhID0gaW50X3JhdGUgfiAuLCBkYXRhID0gaW1wLnRyYWluX2RhdGEsIG51bS50cmVlcyA9IDUwMCwgdmVyYm9zZT1UUlVFLCBpbXBvcnRhbmNlID0gImltcHVyaXR5Iiwgb29iLmVycm9yID0gVFJVRSkKCiMgcHJpbnQgdGhlIG1vZGVsIHN1bW1hcnkKcHJpbnQoIlJhbmRvbSBGb3Jlc3QgTW9kZWwgU3VtbWFyeToiKQpwcmludChpbXAucmYpCgojIG1ha2UgcHJlZGljdGlvbnMgb24gdGhlIHRyYWluaW5nIGFuZCB0ZXN0aW5nIGRhdGEKaW1wLnJmLnRyYWluX3ByZWRpY3Rpb25zIDwtIHByZWRpY3QoaW1wLnJmLCBkYXRhID0gaW1wLnRyYWluX2RhdGEpCmltcC5yZi50ZXN0X3ByZWRpY3Rpb25zIDwtIHByZWRpY3QoaW1wLnJmLCBkYXRhID0gaW1wLnRlc3RfZGF0YSkKCiMgY2FsY3VsYXRlIE1lYW4gU3F1YXJlZCBFcnJvciAoTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKaW1wLnJmLnRyYWluX21zZSA8LSBtZWFuKChpbXAucmYudHJhaW5fcHJlZGljdGlvbnMkcHJlZGljdGlvbnMgLSBpbXAudHJhaW5fZGF0YSRpbnRfcmF0ZSleMikKaW1wLnJmLnRlc3RfbXNlIDwtIG1lYW4oKGltcC5yZi50ZXN0X3ByZWRpY3Rpb25zJHByZWRpY3Rpb25zIC0gaW1wLnRlc3RfZGF0YSRpbnRfcmF0ZSleMikKCiMgY2FsY3VsYXRlIFJvb3QgTWVhbiBTcXVhcmVkIEVycm9yIChSTVNFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKaW1wLnJmLnRyYWluX3Jtc2UgPC0gc3FydChpbXAucmYudHJhaW5fbXNlKQppbXAucmYudGVzdF9ybXNlIDwtIHNxcnQoaW1wLnJmLnRlc3RfbXNlKQoKIyBjYWxjdWxhdGUgTWVhbiBBYnNvbHV0ZSBFcnJvciAoTUFFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKaW1wLnJmLnRyYWluX21hZSA8LSBtZWFuKGFicyhpbXAucmYudHJhaW5fcHJlZGljdGlvbnMkcHJlZGljdGlvbnMgLSBpbXAudHJhaW5fZGF0YSRpbnRfcmF0ZSkpCmltcC5yZi50ZXN0X21hZSA8LSBtZWFuKGFicyhpbXAucmYudGVzdF9wcmVkaWN0aW9ucyRwcmVkaWN0aW9ucyAtIGltcC50ZXN0X2RhdGEkaW50X3JhdGUpKQoKIyBjYWxjdWxhdGUgUi1zcXVhcmVkIChSwrIpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwppbXAucmYudHJhaW5fcjIgPC0gMSAtIChzdW0oKGltcC50cmFpbl9kYXRhJGludF9yYXRlIC0gaW1wLnJmLnRyYWluX3ByZWRpY3Rpb25zJHByZWRpY3Rpb25zKV4yKSAvIHN1bSgoaW1wLnRyYWluX2RhdGEkaW50X3JhdGUgLSBtZWFuKGltcC50cmFpbl9kYXRhJGludF9yYXRlKSleMikpCmltcC5yZi50ZXN0X3IyIDwtIDEgLSAoc3VtKCh0ZXN0X2RhdGEkaW50X3JhdGUgLSByZi50ZXN0X3ByZWRpY3Rpb25zJHByZWRpY3Rpb25zKV4yKSAvIHN1bSgoaW1wLnRlc3RfZGF0YSRpbnRfcmF0ZSAtIG1lYW4oaW1wLnRlc3RfZGF0YSRpbnRfcmF0ZSkpXjIpKQoKIyBkaXNwbGF5IHRoZSBtZXRyaWNzCmNhdCgiVHJhaW5pbmcgTVNFOiIsIGltcC5yZi50cmFpbl9tc2UsICJcbiIpCmNhdCgiVGVzdGluZyBNU0U6IiwgaW1wLnJmLnRlc3RfbXNlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFJNU0U6IiwgaW1wLnJmLnRyYWluX3Jtc2UsICJcbiIpCmNhdCgiVGVzdGluZyBSTVNFOiIsIGltcC5yZi50ZXN0X3Jtc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgTUFFOiIsIGltcC5yZi50cmFpbl9tYWUsICJcbiIpCmNhdCgiVGVzdGluZyBNQUU6IiwgaW1wLnJmLnRlc3RfbWFlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFItc3F1YXJlZCAoUsKyKToiLCBpbXAucmYudHJhaW5fcjIsICJcbiIpCmNhdCgiVGVzdGluZyBSLXNxdWFyZWQgKFLCsik6IiwgaW1wLnJmLnRlc3RfcjIsICJcbiIpCmBgYAoKYGBge3J9CiMjIyMgQm9vc3Rpbmcgd2l0aCBvbmx5IGltcG9ydGFuY2UgdmFyaWFibGVzICMjIyMKCiMgZGVmaW5lIHRoZSB0YXJnZXQgdmFyaWFibGUgZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nCmltcC54Z2IueV90cmFpbiA8LSBpbXAudHJhaW5fZGF0YSRpbnRfcmF0ZQppbXAueGdiLnlfdGVzdCA8LSBpbXAudGVzdF9kYXRhJGludF9yYXRlCgojIGRlZmluZSB0aGUgZmVhdHVyZSBtYXRyaXggZm9yIHRyYWluaW5nIGFuZCB0ZXN0aW5nIChleGNsdWRlIHRoZSB0YXJnZXQgdmFyaWFibGUpCmltcC54Z2IuWF90cmFpbiA8LSBpbXAudHJhaW5fZGF0YVssIC13aGljaChuYW1lcyhpbXAudHJhaW5fZGF0YSkgPT0gJ2ludF9yYXRlJyldCmltcC54Z2IuWF90ZXN0IDwtIGltcC50ZXN0X2RhdGFbLCAtd2hpY2gobmFtZXMoaW1wLnRlc3RfZGF0YSkgPT0gJ2ludF9yYXRlJyldCgojIGZpdCBhIGdyYWRpZW50IGJvb3N0aW5nIHJlZ3Jlc3Npb24gbW9kZWwgdXNpbmcgeGdib29zdAppbXAueGdiIDwtIHhnYm9vc3QoCiAgZGF0YSA9IGFzLm1hdHJpeChpbXAueGdiLlhfdHJhaW4pLAogIGxhYmVsID0gaW1wLnhnYi55X3RyYWluLAogIG5yb3VuZHMgPSAxMDAsCiAgdmVyYm9zZSA9IDAKKQoKIyBtYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0cmFpbmluZyBhbmQgdGVzdGluZyBkYXRhCmltcC54Z2IudHJhaW5fcHJlZGljdGlvbnMgPC0gcHJlZGljdChpbXAueGdiLCBuZXdkYXRhID0gYXMubWF0cml4KGltcC54Z2IuWF90cmFpbikpCmltcC54Z2IudGVzdF9wcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGltcC54Z2IsIG5ld2RhdGEgPSBhcy5tYXRyaXgoaW1wLnhnYi5YX3Rlc3QpKQoKIyBjYWxjdWxhdGUgTWVhbiBTcXVhcmVkIEVycm9yIChNU0UpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwppbXAueGdiLnRyYWluX21zZSA8LSBtZWFuKChpbXAueGdiLnRyYWluX3ByZWRpY3Rpb25zIC0gaW1wLnhnYi55X3RyYWluKV4yKQppbXAueGdiLnRlc3RfbXNlIDwtIG1lYW4oKGltcC54Z2IudGVzdF9wcmVkaWN0aW9ucyAtIGltcC54Z2IueV90ZXN0KV4yKQoKIyBjYWxjdWxhdGUgUm9vdCBNZWFuIFNxdWFyZWQgRXJyb3IgKFJNU0UpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwppbXAueGdiLnRyYWluX3Jtc2UgPC0gc3FydChpbXAueGdiLnRyYWluX21zZSkKaW1wLnhnYi50ZXN0X3Jtc2UgPC0gc3FydChpbXAueGdiLnRlc3RfbXNlKQoKIyBjYWxjdWxhdGUgTWVhbiBBYnNvbHV0ZSBFcnJvciAoTUFFKSBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcKaW1wLnhnYi50cmFpbl9tYWUgPC0gbWVhbihhYnMoaW1wLnhnYi50cmFpbl9wcmVkaWN0aW9ucyAtIGltcC54Z2IueV90cmFpbikpCmltcC54Z2IudGVzdF9tYWUgPC0gbWVhbihhYnMoaW1wLnhnYi50ZXN0X3ByZWRpY3Rpb25zIC0gaW1wLnhnYi55X3Rlc3QpKQoKIyBjYWxjdWxhdGUgUi1zcXVhcmVkIChSwrIpIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZwppbXAueGdiLnRyYWluX3IyIDwtIDEgLSAoc3VtKChpbXAueGdiLnlfdHJhaW4gLSBpbXAueGdiLnRyYWluX3ByZWRpY3Rpb25zKV4yKSAvIHN1bSgoaW1wLnhnYi55X3RyYWluIC0gbWVhbihpbXAueGdiLnlfdHJhaW4pKV4yKSkKaW1wLnhnYi50ZXN0X3IyIDwtIDEgLSAoc3VtKChpbXAueGdiLnlfdGVzdCAtIGltcC54Z2IudGVzdF9wcmVkaWN0aW9ucyleMikgLyBzdW0oKGltcC54Z2IueV90ZXN0IC0gbWVhbihpbXAueGdiLnlfdGVzdCkpXjIpKQoKIyBkaXNwbGF5IHRoZSBtZXRyaWNzCmNhdCgiVHJhaW5pbmcgTVNFOiIsIGltcC54Z2IudHJhaW5fbXNlLCAiXG4iKQpjYXQoIlRlc3RpbmcgTVNFOiIsIGltcC54Z2IudGVzdF9tc2UsICJcbiIpCmNhdCgiVHJhaW5pbmcgUk1TRToiLCBpbXAueGdiLnRyYWluX3Jtc2UsICJcbiIpCmNhdCgiVGVzdGluZyBSTVNFOiIsIGltcC54Z2IudGVzdF9ybXNlLCAiXG4iKQpjYXQoIlRyYWluaW5nIE1BRToiLCBpbXAueGdiLnRyYWluX21hZSwgIlxuIikKY2F0KCJUZXN0aW5nIE1BRToiLCBpbXAueGdiLnRlc3RfbWFlLCAiXG4iKQpjYXQoIlRyYWluaW5nIFItc3F1YXJlZCAoUsKyKToiLCBpbXAueGdiLnRyYWluX3IyLCAiXG4iKQpjYXQoIlRlc3RpbmcgUi1zcXVhcmVkIChSwrIpOiIsIGltcC54Z2IudGVzdF9yMiwgIlxuIikKYGBgCgpUaGUgZGF0YXNldCB3YXMgZmlsdGVyZWQgYnkgdGhlIDIwIHZhcmlhYmxlcyB3aXRoIHRoZSBtb3N0IGltcG9ydGFuY2UgKGZyb20gdGhlIHJmIHJlc3VsdHMpLiBBcyB3ZSBjYW4gc2VlIGFib3ZlLCB0aGUgZXJyb3JzIG9mIGVhY2ggbW9kZWwgYXJlIG1vcmUgb3IgbGVzcyB0aGUgZXJyb3JzIHdpdGggdGhlIGRvdWJsZSB2YXJpYWJsZXMgd2UgaGFkIGJlZm9yZSwgc28gZmlsdGVyaW5nIGJ5IHRoZXNlIDIwICJpbXBvcnRhbnQgdmFyaWFibGVzIiBkb2VzIG5vdCBzZWVtIG1ha2luZyBzZW5zZS4uLgoKSHlwZXJwYXJhbWV0ZXIgVHVuaW5nIGZvciBYR0Jvb3N0aW5nOgoKYGBge3J9CiMgZGVmaW5lIHRoZSBudW1iZXIgb2YgY29yZXMKbnVtQ29yZXMgPC0gZGV0ZWN0Q29yZXMoKSAtIDEKCiMgcmVnaXN0ZXIgZG9QYXJhbGxlbCBhcyB0aGUgYmFja2VuZCBmb3IgcGFyYWxsZWwgZXhlY3V0aW9uCnJlZ2lzdGVyRG9QYXJhbGxlbChjb3Jlcz1udW1Db3JlcykKCiMgZGVmaW5lIHRoZSBjb250cm9sIHVzaW5nIGEgY3Jvc3MtdmFsaWRhdGlvbiBhcHByb2FjaAp0cmFpbl9jb250cm9sIDwtIHRyYWluQ29udHJvbChtZXRob2QgPSAiY3YiLCBudW1iZXIgPSA1LCB2ZXJib3NlSXRlciA9IFRSVUUpCgojIGRlZmluZSB0aGUgZ3JpZCBvZiBoeXBlcnBhcmFtZXRlcnMgdG8gc2VhcmNoIG92ZXIKeGdiLmdyaWQgPC0gZXhwYW5kLmdyaWQoCiAgbnJvdW5kcyA9IGMoMTAwLCAyMDAsIDMwMCksCiAgZXRhID0gYygwLjAxLCAwLjA1LCAwLjEpLAogIG1heF9kZXB0aCA9IGMoMywgNiwgOSksCiAgZ2FtbWEgPSBjKDAsIDAuMSwgMC4yKSwKICBjb2xzYW1wbGVfYnl0cmVlID0gYygwLjUsIDAuOCwgMSksCiAgbWluX2NoaWxkX3dlaWdodCA9IGMoMSwgNSwgMTApLAogIHN1YnNhbXBsZSA9IGMoMC41LCAwLjc1LCAxKQopCgojIHRyYWluIHRoZSBtb2RlbAp4Z2IudHVuZWQgPC0gdHJhaW4oCiAgeCA9IHRyYWluX2RhdGEsIHkgPSB4Z2IueV90cmFpbiwKICBtZXRob2QgPSAieGdiVHJlZSIsCiAgdHJDb250cm9sID0gdHJhaW5fY29udHJvbCwKICB0dW5lR3JpZCA9IHhnYi5ncmlkCikKCiMgdmlldyB0aGUgYmVzdCB0dW5pbmcgcGFyYW1ldGVycwpwcmludCh4Z2IudHVuZWQkYmVzdFR1bmUpCgojIHN0b3AgdGhlIHBhcmFsbGVsIGJhY2tlbmQKc3RvcEltcGxpY2l0Q2x1c3RlcigpCmBgYAoKU2F2aW5nIG91ciBiZXN0IG1vZGVsOgoKYGBge3J9CnNhdmVSRFMoeGdiLCBmaWxlID0gImJlc3RfbW9kZWxfeGdiLnJkcyIpCmBgYAoK